Unit test get_bid_ask_realtime_raw_data_reader
Opened this issue · 5 comments
Lets add unit test for this method
https://github.com/kaizen-ai/kaizenflow/blob/423470c58122af11b12a1b9c3ea889abcad52342/im_v2/common/data/client/im_raw_data_client.py#L373
Test if bid_ask_db_signature is correctly built from the params.
Input params: stage = 'preprod', data_vendor = 'MOCK_VENDOR', universe_version = 'v7_5`
FYI @samarth9008
I encountered an Assertion error in parsing the signature:
Traceback (most recent call last):
File "/usr/lib/python3.9/unittest/case.py", line 59, in testPartExecutor
yield
File "/usr/lib/python3.9/unittest/case.py", line 593, in run
self._callTestMethod(testMethod)
File "/usr/lib/python3.9/unittest/case.py", line 550, in _callTestMethod
method()
File "/app/im_v2/common/data/client/test/test_im_raw_data_client.py", line 1158, in test1
actual = imvcdcimrdc.get_bid_ask_realtime_raw_data_reader(
File "/app/im_v2/common/data/client/im_raw_data_client.py", line 391, in get_bid_ask_realtime_raw_data_reader
bid_ask_raw_data_reader = RawDataReader(bid_ask_db_signature, stage=stage)
File "/app/im_v2/common/data/client/im_raw_data_client.py", line 45, in __init__
self.args = dsdascut.parse_dataset_signature_to_args(
File "/app/data_schema/dataset_schema_utils.py", line 258, in parse_dataset_signature_to_args
hdbg.dassert_eq(validate_dataset_signature(signature, dataset_schema), True)
File "/app/helpers/hdbg.py", line 172, in dassert_eq
_dfatal(txt, msg, *args, only_warning=only_warning)
File "/app/helpers/hdbg.py", line 142, in _dfatal
dfatal(dfatal_txt)
File "/app/helpers/hdbg.py", line 71, in dfatal
raise assertion_type(ret)
AssertionError:
################################################################################
* Failed assertion *
'False'
==
'True'
################################################################################
@smitpatel49 did you try to debug why the assertion was raised?
I tried and I think that validate_dataset_signature() does not produce the schema we need. I tried using 'CCXT' as input parameter but I don't think I have the aws credentials.
For the input parameters:
stage = 'preprod'
data_vendor = 'MOCK_VENDOR'
universe_version = 'v7_5'
exchange_id = 'binance'
I'm getting the following error:
____________________________________________________________________________ ERROR collecting research_amp/soccer_prediction/paired_t_test.py _____________________________________________________________________________
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/_pytest/runner.py", line 341, in from_call
result: Optional[TResult] = func()
File "/venv/lib/python3.9/site-packages/_pytest/runner.py", line 372, in
call = CallInfo.from_call(lambda: list(collector.collect()), "collect")
File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 531, in collect
self._inject_setup_module_fixture()
File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 545, in _inject_setup_module_fixture
self.obj, ("setUpModule", "setup_module")
File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 310, in obj
self._obj = obj = self._getobj()
File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 528, in _getobj
return self._importtestmodule()
File "/venv/lib/python3.9/site-packages/_pytest/python.py", line 617, in _importtestmodule
mod = import_path(self.path, mode=importmode, root=self.config.rootpath)
File "/venv/lib/python3.9/site-packages/_pytest/pathlib.py", line 567, in import_path
importlib.import_module(module_name)
File "/usr/lib/python3.9/importlib/init.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "", line 1030, in _gcd_import
File "", line 1007, in _find_and_load
File "", line 986, in _find_and_load_unlocked
File "", line 680, in _load_unlocked
File "/venv/lib/python3.9/site-packages/_pytest/assertion/rewrite.py", line 178, in exec_module
exec(co, module.dict)
File "/app/research_amp/soccer_prediction/paired_t_test.py", line 104, in
glm_predictions = get_predictions_data(bucket_name=bucket_name, dataset_path=s3_file_path)
File "/app/research_amp/soccer_prediction/paired_t_test.py", line 62, in get_predictions_data
s3 = haws.get_service_resource(aws_profile="ck", service_name="s3")
File "/app/helpers/haws.py", line 75, in get_service_resource
session = get_session(aws_profile)
File "/app/helpers/haws.py", line 44, in get_session
credentials = hs3.get_aws_credentials(aws_profile=aws_profile)
File "/app/helpers/hs3.py", line 723, in get_aws_credentials
config = _get_aws_config(file_name)
File "/app/helpers/hs3.py", line 520, in _get_aws_config
hdbg.dassert_file_exists(file_name)
File "/app/helpers/hdbg.py", line 785, in dassert_file_exists
_dfatal(txt, msg, *args, only_warning=only_warning)
File "/app/helpers/hdbg.py", line 142, in _dfatal
dfatal(dfatal_txt)
File "/app/helpers/hdbg.py", line 71, in dfatal
raise assertion_type(ret)
AssertionError:
################################################################################
- Failed assertion *
File '/home/.aws/credentials' doesn't exist
################################################################################
The test accesses the AWS credentials that don't exist in the specified path. @sonaalKant any suggestions on how this can be handled?
@sonaalKant as I don't have the file '/home/.aws/credentials', am I supposed to use a library like moto to mock AWS services for testing purposes?