chrism2671/PyTrendFollow

Some multiprocessing issue

cwengc opened this issue · 5 comments

cwengc commented

multiprocessing_on_dill.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexes\base.py", line 2525, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 117, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 139, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1265, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1273, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'open'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 44, in mapstar
return list(map(*args))
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 61, in
self.memo_inst_calc = dict(pool.map(lambda x: (x.name, x.calculate()), self.portfolio))
File "C:/Code/PyTrendFollow-master\core\instrument.py", line 82, in calculate
rate = pd.Series(1, index=self.panama_prices().index)
File "C:/Code/PyTrendFollow-master\core\instrument.py", line 170, in panama_prices
return self.contracts()['close'].diff().to_frame().swaplevel().fillna(0).join(
File "C:/Code/PyTrendFollow-master\core\instrument.py", line 284, in contracts
data = data[(data['open'] > 0) & (data['volume'] > 1)]
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py", line 2139, in getitem
return self._getitem_column(key)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py", line 2146, in _getitem_column
return self._get_item_cache(key)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\generic.py", line 1842, in _get_item_cache
values = self._data.get(item)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py", line 3843, in get
loc = self.items.get_loc(item)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexes\base.py", line 2527, in get_loc
return self._engine.get_loc(self._maybe_cast_indexer(key))
File "pandas/_libs/index.pyx", line 117, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 139, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1265, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1273, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'open'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "C:/Code/PyTrendFollow-master/scheduler.py", line 111, in
main()
File "C:/Code/PyTrendFollow-master/scheduler.py", line 97, in main
schedule.run_pending()
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init_.py", line 493, in run_pending
default_scheduler.run_pending()
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init_.py", line 78, in run_pending
self.run_job(job)
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init.py", line 131, in run_job
ret = job.run()
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init.py", line 411, in run
ret = self.job_func()
File "C:/Code/PyTrendFollow-master/scheduler.py", line 51, in sync_trades
ib.sync_portfolio(p, acc=a, trade=trade)
File "C:/Code/PyTrendFollow-master\trading\ibstate.py", line 167, in sync_portfolio
frontier = portfolio.frontier(capital=acc.net)
File "C:/Code/PyTrendFollow-master\trading\portfolio.py", line 229, in frontier
c = self.curve(capital=capital)
File "C:/Code/PyTrendFollow-master\trading\portfolio.py", line 42, in curve
return accountCurve(list(self.valid_instruments().values()), **kw2)
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 35, in init
self.positions = self.instrument_positions()
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 71, in instrument_positions
self.memo_instrument_positions = pd.DataFrame({k: v['position'] for k, v in self.inst_calc().items()})
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 61, in inst_calc
self.memo_inst_calc = dict(pool.map(lambda x: (x.name, x.calculate()), self.portfolio))
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 260, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 608, in get
raise self._value
KeyError: 'open'
2018-03-13 04:22:49,332 - scheduler - ERROR - 'open'
multiprocessing_on_dill.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexes\base.py", line 2525, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 117, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 139, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1265, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1273, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'open'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 44, in mapstar
return list(map(*args))
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 61, in
self.memo_inst_calc = dict(pool.map(lambda x: (x.name, x.calculate()), self.portfolio))
File "C:/Code/PyTrendFollow-master\core\instrument.py", line 82, in calculate
rate = pd.Series(1, index=self.panama_prices().index)
File "C:/Code/PyTrendFollow-master\core\instrument.py", line 170, in panama_prices
return self.contracts()['close'].diff().to_frame().swaplevel().fillna(0).join(
File "C:/Code/PyTrendFollow-master\core\instrument.py", line 284, in contracts
data = data[(data['open'] > 0) & (data['volume'] > 1)]
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py", line 2139, in getitem
return self._getitem_column(key)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py", line 2146, in _getitem_column
return self._get_item_cache(key)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\generic.py", line 1842, in _get_item_cache
values = self._data.get(item)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py", line 3843, in get
loc = self.items.get_loc(item)
File "C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexes\base.py", line 2527, in get_loc
return self._engine.get_loc(self._maybe_cast_indexer(key))
File "pandas/_libs/index.pyx", line 117, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 139, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1265, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1273, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'open'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "C:/Code/PyTrendFollow-master/scheduler.py", line 111, in
main()
File "C:/Code/PyTrendFollow-master/scheduler.py", line 97, in main
schedule.run_pending()
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init_.py", line 493, in run_pending
default_scheduler.run_pending()
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init_.py", line 78, in run_pending
self.run_job(job)
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init.py", line 131, in run_job
ret = job.run()
File "C:\ProgramData\Anaconda3\lib\site-packages\schedule_init.py", line 411, in run
ret = self.job_func()
File "C:/Code/PyTrendFollow-master/scheduler.py", line 51, in sync_trades
ib.sync_portfolio(p, acc=a, trade=trade)
File "C:/Code/PyTrendFollow-master\trading\ibstate.py", line 167, in sync_portfolio
frontier = portfolio.frontier(capital=acc.net)
File "C:/Code/PyTrendFollow-master\trading\portfolio.py", line 229, in frontier
c = self.curve(capital=capital)
File "C:/Code/PyTrendFollow-master\trading\portfolio.py", line 42, in curve
return accountCurve(list(self.valid_instruments().values()), **kw2)
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 35, in init
self.positions = self.instrument_positions()
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 71, in instrument_positions
self.memo_instrument_positions = pd.DataFrame({k: v['position'] for k, v in self.inst_calc().items()})
File "C:/Code/PyTrendFollow-master\trading\accountcurve.py", line 61, in inst_calc
self.memo_inst_calc = dict(pool.map(lambda x: (x.name, x.calculate()), self.portfolio))
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 260, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\ProgramData\Anaconda3\lib\site-packages\multiprocessing_on_dill\pool.py", line 608, in get
raise self._value
KeyError: 'open'
Could not find thread pid_8636_id_202783656
Available: ['pid_8636_id_106792608', 'pid_8636_id_106791600', 'pid_8636_id_106792328', 'pid_8636_id_202818560', 'pid_8636_id_202741464', 'pid_8636_id_106792160', 'pid_8636_id_183354256', 'pid_8636_id_18849408']
Could not find thread pid_8636_id_202817776
Available: ['pid_8636_id_106792608', 'pid_8636_id_106791600', 'pid_8636_id_106792328', 'pid_8636_id_202818560', 'pid_8636_id_202741464', 'pid_8636_id_106792160', 'pid_8636_id_183354256', 'pid_8636_id_18849408']
Could not find thread pid_8636_id_106792720
Available: ['pid_8636_id_106792608', 'pid_8636_id_106791600', 'pid_8636_id_106792328', 'pid_8636_id_202818560', 'pid_8636_id_202741464', 'pid_8636_id_106792160', 'pid_8636_id_183354256', 'pid_8636_id_18849408']

Process finished with exit code 0

ka-petrov commented

Can you please provide the following info:

  1. What Windows version are you using?
  2. Your Python version
  3. List your Python packages versions (run pip freeze in the cmd and copy-paste the output here)

As we haven't really tested this project on Windows yet, there is a pretty good chance this issue is OS-specific. We'll try to find out anyway, thanks for posting!

cwengc commented

Python 3.6
Win7

alabaster==0.7.10
anaconda-client==1.6.11
anaconda-navigator==1.7.0
anaconda-project==0.8.2
arch==4.3.1
asn1crypto==0.24.0
astroid==1.6.1
astropy==3.0
attrs==17.4.0
Babel==2.5.3
backports.shutil-get-terminal-size=
beautifulsoup4==4.6.0
bitarray==0.8.1
bkcharts==0.2
blaze==0.11.3
bleach==2.1.3
bokeh==0.12.14
boto==2.48.0
Bottleneck==1.2.1
bravado==9.2.2
bravado-core==4.13.1
CacheControl==0.12.4
certifi==2018.1.18
cffi==1.11.5
chardet==3.0.4
click==6.7
cloudpickle==0.5.2
clyent==1.2.2
colorama==0.3.9
comtypes==1.1.4
conda==4.3.34
conda-build==3.7.1
conda-verify==2.0.0
contextlib2==0.5.5
cryptography==2.1.4
cycler==0.10.0
Cython==0.27.3
cytoolz==0.9.0.1
dask==0.17.1
datashape==0.5.4
decorator==4.2.1
dill==0.2.7.1
distlib==0.2.6
distributed==1.21.2
docutils==0.14
entrypoints==0.2.3
et-xmlfile==1.0.1
fastcache==1.0.2
filelock==3.0.4
Flask==0.12.2
Flask-Cors==3.0.3
gevent==1.2.2
glob2==0.6
greenlet==0.4.13
h5py==2.7.1
heapdict==1.0.0
html5lib==1.0.1
ib-insync==0.9.2
ibapi==9.73.6
IbPy2==0.8.0
idna==2.6
imageio==2.2.0
imagesize==1.0.0
inflection==0.3.1
ipykernel==4.8.2
ipython==6.2.1
ipython-genutils==0.2.0
ipywidgets==7.1.2
isort==4.3.4
itsdangerous==0.24
jdcal==1.3
jedi==0.11.1
Jinja2==2.10
jsonref==0.1
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.2.2
jupyter-console==5.2.0
jupyter-core==4.4.0
jupyterlab==0.31.12
jupyterlab-launcher==0.10.5
lazy-object-proxy==1.3.1
llvmlite==0.22.0
locket==0.2.0
lockfile==0.12.2
lxml==4.1.1
MarkupSafe==1.0
matplotlib==2.1.2
mccabe==0.6.1
menuinst==1.4.11
mistune==0.8.3
more-itertools==4.1.0
mpmath==1.0.0
msgpack-python==0.5.5
multipledispatch==0.5.0
multiprocessing-on-dill==3.5.0a4
mysql-connector==2.1.6
navigator-updater==0.1.0
nbconvert==5.3.1
nbformat==4.4.0
networkx==2.1
nltk==3.2.5
nose==1.3.7
notebook==5.4.0
numba==0.37.0
numexpr==2.6.2
numpy==1.14.1
numpydoc==0.7.0
odo==0.5.1
olefile==0.45.1
openpyxl==2.5.0
packaging==17.1
pandas==0.22.0
pandocfilters==1.4.2
parso==0.1.1
partd==0.3.8
path.py==11.0
pathlib2==2.3.0
patsy==0.5.0
pep8==1.7.1
pickleshare==0.7.4
Pillow==4.2.1
pkginfo==1.4.1
pluggy==0.6.0
ply==3.11
progress==1.3
prompt-toolkit==1.0.15
psutil==5.4.3
py==1.5.2
pycodestyle==2.3.1
pycosat==0.6.3
pycparser==2.18
pycrypto==2.6.1
pycurl==7.43.0.1
pyflakes==1.6.0
Pygments==2.2.0
pylint==1.8.2
pymongo==3.4.0
pyodbc==4.0.22
pyOpenSSL==17.5.0
pyparsing==2.2.0
PyPrind==2.10.0
pyreadline==2.1
PySocks==1.6.8
pytest==3.4.2
pytest-arraydiff==0.2
pytest-astropy==0.2.1
pytest-doctestplus==0.1.2
pytest-openfiles==0.2.0
pytest-remotedata==0.2.0
python-dateutil==2.6.1
pytz==2018.3
PyWavelets==0.5.2
pywin32==222
pywinpty==0.5.1
PyYAML==3.12
pyzmq==17.0.0
QtAwesome==0.4.4
qtconsole==4.3.1
QtPy==1.3.1
Quandl==3.3.0
requests==2.18.4
rfc3987==1.3.7
rope==0.10.7
ruamel-yaml==0.15.35
schedule==0.5.0
scikit-image==0.13.1
scikit-learn==0.19.1
scipy==1.0.0
seaborn==0.8.1
Send2Trash==1.5.0
simplegeneric==0.8.1
simplejson==3.13.2
singledispatch==3.4.0.3
six==1.11.0
snowballstemmer==1.2.1
sortedcollections==0.5.3
sortedcontainers==1.5.9
Sphinx==1.7.1
sphinxcontrib-websupport==1.0.1
spyder==3.2.7
SQLAlchemy==1.2.0
statsmodels==0.8.0
strict-rfc3339==0.7
swagger-spec-validator==2.1.0
sympy==1.1.1
tables==3.4.2
tblib==1.3.2
terminado==0.8.1
testpath==0.3.1
toolz==0.9.0
tornado==5.0
traitlets==4.3.2
typing==3.6.4
unicodecsv==0.14.1
urllib3==1.22
wcwidth==0.1.7
webcolors==1.8.1
webencodings==0.5.1
Werkzeug==0.14.1
widgetsnbextension==3.1.4
win-inet-pton==1.0.1
win-unicode-console==0.5
wincertstore==0.2
wrapt==1.10.11
xlrd==1.1.0
XlsxWriter==1.0.2
xlwings==0.11.7
xlwt==1.3.0
zict==0.1.3

chrism2671 commented

This looks like a missing data issue to me - KeyError: 'open'

chrism2671 commented

@cwengc Is that error from running ./validate.py?

cwengc commented

Realised that the error was caused by missing GXBT data. Commented out GXBT for now.