dump_session failing in jupyter lab
Closed this issue · 2 comments
The following simple example is failing:
import dill
a = 1
dill.dump_session("nb.db")
with the following error relating to pickling '_hashlib.HMAC' object:
TypeError Traceback (most recent call last)
Input In [4], in
1 get_ipython().run_line_magic('reset', 'out')
2 ipython = get_ipython()
----> 3 dill.dump_session('notebook_env.db')File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:503, in dump_session(filename, main, byref, **kwds)
501 pickler._first_pass = True
502 pickler._main_modified = main is not pickler._original_main
--> 503 pickler.dump(main)
504 finally:
505 if f is not filename: # If newly opened fileFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:620, in Pickler.dump(self, obj)
618 raise PicklingError(msg)
619 else:
--> 620 StockPickler.dump(self, obj)
621 returnFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:487, in _Pickler.dump(self, obj)
485 if self.proto >= 4:
486 self.framer.start_framing()
--> 487 self.save(obj)
488 self.write(STOP)
489 self.framer.end_framing()File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:560, in _Pickler.save(self, obj, save_persistent_id)
558 f = self.dispatch.get(t)
559 if f is not None:
--> 560 f(self, obj) # Call unbound method with explicit self
561 return
563 # Check private dispatch table if any, or else
564 # copyreg.dispatch_tableFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:1765, in save_module(pickler, obj)
1762 _main_dict = obj.dict.copy() #XXX: better no copy? option to copy?
1763 [_main_dict.pop(item, None) for item in singletontypes
1764 + ["builtins", "loader"]]
-> 1765 pickler.save_reduce(_import_module, (obj.name,), obj=obj,
1766 state=_main_dict)
1767 log.info("# M1")
1768 elif PY3 and obj.name == "dill._dill":File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:717, in _Pickler.save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
715 if state is not None:
716 if state_setter is None:
--> 717 save(state)
718 write(BUILD)
719 else:
720 # If a state_setter is specified, call it instead of load_build
721 # to update obj's with its previous state.
722 # First, push state_setter and its tuple of expected arguments
723 # (obj, state) onto the stack.File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:560, in _Pickler.save(self, obj, save_persistent_id)
558 f = self.dispatch.get(t)
559 if f is not None:
--> 560 f(self, obj) # Call unbound method with explicit self
561 return
563 # Check private dispatch table if any, or else
564 # copyreg.dispatch_tableFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:1251, in save_module_dict(pickler, obj)
1248 if is_dill(pickler, child=False) and pickler._session:
1249 # we only care about session the first pass thru
1250 pickler._first_pass = False
-> 1251 StockPickler.save_dict(pickler, obj)
1252 log.info("# D2")
1253 returnFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:972, in _Pickler.save_dict(self, obj)
969 self.write(MARK + DICT)
971 self.memoize(obj)
--> 972 self._batch_setitems(obj.items())File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:998, in _Pickler._batch_setitems(self, items)
996 for k, v in tmp:
997 save(k)
--> 998 save(v)
999 write(SETITEMS)
1000 elif n:File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:603, in _Pickler.save(self, obj, save_persistent_id)
599 raise PicklingError("Tuple returned by %s must have "
600 "two to six elements" % reduce)
602 # Save the reduce() output and finally memoize the object
--> 603 self.save_reduce(obj=obj, *rv)File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:717, in _Pickler.save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
715 if state is not None:
716 if state_setter is None:
--> 717 save(state)
718 write(BUILD)
719 else:
720 # If a state_setter is specified, call it instead of load_build
721 # to update obj's with its previous state.
722 # First, push state_setter and its tuple of expected arguments
723 # (obj, state) onto the stack.File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:560, in _Pickler.save(self, obj, save_persistent_id)
558 f = self.dispatch.get(t)
559 if f is not None:
--> 560 f(self, obj) # Call unbound method with explicit self
561 return
563 # Check private dispatch table if any, or else
564 # copyreg.dispatch_tableFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:1251, in save_module_dict(pickler, obj)
1248 if is_dill(pickler, child=False) and pickler._session:
1249 # we only care about session the first pass thru
1250 pickler._first_pass = False
-> 1251 StockPickler.save_dict(pickler, obj)
1252 log.info("# D2")
1253 returnFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:972, in _Pickler.save_dict(self, obj)
969 self.write(MARK + DICT)
971 self.memoize(obj)
--> 972 self._batch_setitems(obj.items())File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:998, in _Pickler._batch_setitems(self, items)
996 for k, v in tmp:
997 save(k)
--> 998 save(v)
999 write(SETITEMS)
1000 elif n:File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:603, in _Pickler.save(self, obj, save_persistent_id)
599 raise PicklingError("Tuple returned by %s must have "
600 "two to six elements" % reduce)
602 # Save the reduce() output and finally memoize the object
--> 603 self.save_reduce(obj=obj, *rv)[... skipping similar frames: _Pickler.save at line 560 (8 times), _Pickler._batch_setitems at line 998 (7 times), _Pickler.save_dict at line 972 (7 times), save_module_dict at line 1251 (7 times), _Pickler.save_reduce at line 717 (4 times), _Pickler.save at line 603 (3 times)]File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:1251, in save_module_dict(pickler, obj)
1248 if is_dill(pickler, child=False) and pickler._session:
1249 # we only care about session the first pass thru
1250 pickler._first_pass = False
-> 1251 StockPickler.save_dict(pickler, obj)
1252 log.info("# D2")
1253 returnFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:972, in _Pickler.save_dict(self, obj)
969 self.write(MARK + DICT)
971 self.memoize(obj)
--> 972 self._batch_setitems(obj.items())File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:998, in _Pickler._batch_setitems(self, items)
996 for k, v in tmp:
997 save(k)
--> 998 save(v)
999 write(SETITEMS)
1000 elif n:File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:603, in _Pickler.save(self, obj, save_persistent_id)
599 raise PicklingError("Tuple returned by %s must have "
600 "two to six elements" % reduce)
602 # Save the reduce() output and finally memoize the object
--> 603 self.save_reduce(obj=obj, *rv)File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:717, in _Pickler.save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
715 if state is not None:
716 if state_setter is None:
--> 717 save(state)
718 write(BUILD)
719 else:
720 # If a state_setter is specified, call it instead of load_build
721 # to update obj's with its previous state.
722 # First, push state_setter and its tuple of expected arguments
723 # (obj, state) onto the stack.File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:560, in _Pickler.save(self, obj, save_persistent_id)
558 f = self.dispatch.get(t)
559 if f is not None:
--> 560 f(self, obj) # Call unbound method with explicit self
561 return
563 # Check private dispatch table if any, or else
564 # copyreg.dispatch_tableFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:887, in _Pickler.save_tuple(self, obj)
885 if n <= 3 and self.proto >= 2:
886 for element in obj:
--> 887 save(element)
888 # Subtle. Same as in the big comment below.
889 if id(obj) in memo:File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:560, in _Pickler.save(self, obj, save_persistent_id)
558 f = self.dispatch.get(t)
559 if f is not None:
--> 560 f(self, obj) # Call unbound method with explicit self
561 return
563 # Check private dispatch table if any, or else
564 # copyreg.dispatch_tableFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/site-packages/dill/_dill.py:1251, in save_module_dict(pickler, obj)
1248 if is_dill(pickler, child=False) and pickler._session:
1249 # we only care about session the first pass thru
1250 pickler._first_pass = False
-> 1251 StockPickler.save_dict(pickler, obj)
1252 log.info("# D2")
1253 returnFile ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:972, in _Pickler.save_dict(self, obj)
969 self.write(MARK + DICT)
971 self.memoize(obj)
--> 972 self._batch_setitems(obj.items())File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:998, in _Pickler._batch_setitems(self, items)
996 for k, v in tmp:
997 save(k)
--> 998 save(v)
999 write(SETITEMS)
1000 elif n:File ~/opt/anaconda3/envs/geoenv3/lib/python3.10/pickle.py:578, in _Pickler.save(self, obj, save_persistent_id)
576 reduce = getattr(obj, "reduce_ex", None)
577 if reduce is not None:
--> 578 rv = reduce(self.proto)
579 else:
580 reduce = getattr(obj, "reduce", None)TypeError: cannot pickle '_hashlib.HMAC' object
Inside IPython (and therefore JupyterLab), there's a bunch of objects referenced in the __main__ module's dictionary, i.e. the interactive namespace. You can list them by calling dir() at the prompt. This problem is known and is dealt with by #475, which is pending review and expected to be included in the following release, to be finished in the next days hopefully.
You can identify the offending object with the following code:
import dill
for name, obj in globals().copy().items():
if not dill.pickles(obj):
print(f"{name}: {type(obj)}")You can disregard get_ipython, exit and quit (these are special cased in dump_session()), and also __builtin__ and __builtins__ (these are flagged due to a bug in dill.pickles() actually). The objects _oh, Out and variables matching _+\d* (ex.: __, _17) contain references to objects of the output history and should not cause problems in a clean new session.