Version 2.0.x broke tavern tests which were working with version 1.25.2
alexbigkid opened this issue · 2 comments
alexbigkid commented
Looks like there was a breaking change with the 2.0 release. Could you please take a look what is breaking in the new release? I am getting following error when running tests based on tavern test framework.
=========================================================================================================== test session starts ============================================================================================================
platform darwin -- Python 3.9.16, pytest-7.2.1, pluggy-1.0.0
rootdir: /
plugins: tavern-2.0.1
collected 1 item
test_accept_valid_fleet_provision.tavern.yml F [100%]
================================================================================================================= FAILURES =================================================================================================================
_______________________________ /Users/alexberger/dev/aai/cloud-infrastructure/integration-tests/fleet-provisioning/test_accept_valid_fleet_provision.tavern.yml::Test accept valid fleet provision request ________________________________
cls = <class '_pytest.runner.CallInfo'>, func = <function call_runtest_hook.<locals>.<lambda> at 0x10bf38b80>, when = 'call', reraise = (<class '_pytest.outcomes.Exit'>, <class 'KeyboardInterrupt'>)
@classmethod
def from_call(
cls,
func: "Callable[[], TResult]",
when: "Literal['collect', 'setup', 'call', 'teardown']",
reraise: Optional[
Union[Type[BaseException], Tuple[Type[BaseException], ...]]
] = None,
) -> "CallInfo[TResult]":
"""Call func, wrapping the result in a CallInfo.
:param func:
The function to call. Called without arguments.
:param when:
The phase in which the function is called.
:param reraise:
Exception or exceptions that shall propagate if raised by the
function, instead of being wrapped in the CallInfo.
"""
excinfo = None
start = timing.time()
precise_start = timing.perf_counter()
try:
> result: Optional[TResult] = func()
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:339:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> lambda: ihook(item=item, **kwds), when=when, reraise=reraise
)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:260:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <_HookCaller 'pytest_runtest_call'>, args = (), kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, argname = 'item', firstresult = False
def __call__(self, *args, **kwargs):
if args:
raise TypeError("hook calling supports only keyword arguments")
assert not self.is_historic()
# This is written to avoid expensive operations when not needed.
if self.spec:
for argname in self.spec.argnames:
if argname not in kwargs:
notincall = tuple(set(self.spec.argnames) - kwargs.keys())
warnings.warn(
"Argument(s) {} which are declared in the hookspec "
"can not be found in this hook call".format(notincall),
stacklevel=2,
)
break
firstresult = self.spec.opts.get("firstresult")
else:
firstresult = False
> return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_hooks.py:265:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <_pytest.config.PytestPluginManager object at 0x10bde3b80>, hook_name = 'pytest_runtest_call'
methods = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False
def _hookexec(self, hook_name, methods, kwargs, firstresult):
# called from all hookcaller instances.
# enable_tracing will set its own wrapping function at self._inner_hookexec
> return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_manager.py:80:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False
def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
"""Execute a call into multiple python functions/methods and return the
result(s).
``caller_kwargs`` comes from _HookCaller.__call__().
"""
__tracebackhide__ = True
results = []
excinfo = None
try: # run impl and wrapper setup functions in a loop
teardowns = []
try:
for hook_impl in reversed(hook_impls):
try:
args = [caller_kwargs[argname] for argname in hook_impl.argnames]
except KeyError:
for argname in hook_impl.argnames:
if argname not in caller_kwargs:
raise HookCallError(
f"hook call must provide argument {argname!r}"
)
if hook_impl.hookwrapper:
try:
gen = hook_impl.function(*args)
next(gen) # first yield
teardowns.append(gen)
except StopIteration:
_raise_wrapfail(gen, "did not yield")
else:
res = hook_impl.function(*args)
if res is not None:
results.append(res)
if firstresult: # halt further impl calls
break
except BaseException:
excinfo = sys.exc_info()
finally:
if firstresult: # first result hooks return a single value
outcome = _Result(results[0] if results else None, excinfo)
else:
outcome = _Result(results, excinfo)
# run all wrapper post-yield blocks
for gen in reversed(teardowns):
try:
gen.send(outcome)
_raise_wrapfail(gen, "has second yield")
except StopIteration:
pass
> return outcome.get_result()
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_callers.py:60:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <pluggy._result._Result object at 0x10beea370>
def get_result(self):
"""Get the result(s) for this hook call.
If the hook was marked as a ``firstresult`` only a single value
will be returned otherwise a list of results.
"""
__tracebackhide__ = True
if self._excinfo is None:
return self._result
else:
ex = self._excinfo
> raise ex[1].with_traceback(ex[2])
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_result.py:60:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
hook_name = 'pytest_runtest_call'
hook_impls = [<HookImpl plugin_name='runner', plugin=<module '_pytest.runner' from '/Users/alexberger/.pyenv/versions/fp/lib/python....threadexception' from '/Users/alexberger/.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/threadexception.py'>>]
caller_kwargs = {'item': <YamlItem Test accept valid fleet provision request>}, firstresult = False
def _multicall(hook_name, hook_impls, caller_kwargs, firstresult):
"""Execute a call into multiple python functions/methods and return the
result(s).
``caller_kwargs`` comes from _HookCaller.__call__().
"""
__tracebackhide__ = True
results = []
excinfo = None
try: # run impl and wrapper setup functions in a loop
teardowns = []
try:
for hook_impl in reversed(hook_impls):
try:
args = [caller_kwargs[argname] for argname in hook_impl.argnames]
except KeyError:
for argname in hook_impl.argnames:
if argname not in caller_kwargs:
raise HookCallError(
f"hook call must provide argument {argname!r}"
)
if hook_impl.hookwrapper:
try:
gen = hook_impl.function(*args)
next(gen) # first yield
teardowns.append(gen)
except StopIteration:
_raise_wrapfail(gen, "did not yield")
else:
> res = hook_impl.function(*args)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/pluggy/_callers.py:39:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
item = <YamlItem Test accept valid fleet provision request>
def pytest_runtest_call(item: Item) -> None:
_update_current_test_var(item, "call")
try:
del sys.last_type
del sys.last_value
del sys.last_traceback
except AttributeError:
pass
try:
item.runtest()
except Exception as e:
# Store trace info to allow postmortem debugging
sys.last_type = type(e)
sys.last_value = e
assert e.__traceback__ is not None
# Skip *this* frame
sys.last_traceback = e.__traceback__.tb_next
> raise e
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:175:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
item = <YamlItem Test accept valid fleet provision request>
def pytest_runtest_call(item: Item) -> None:
_update_current_test_var(item, "call")
try:
del sys.last_type
del sys.last_value
del sys.last_traceback
except AttributeError:
pass
try:
> item.runtest()
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/_pytest/runner.py:167:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <YamlItem Test accept valid fleet provision request>
def runtest(self):
self.global_cfg = load_global_cfg(self.config)
load_plugins(self.global_cfg)
# INTERNAL
xfail = self.spec.get("_xfail", False)
try:
fixture_values = self._load_fixture_values()
self.global_cfg.variables.update(fixture_values)
call_hook(
self.global_cfg,
"pytest_tavern_beta_before_every_test_run",
test_dict=self.spec,
variables=self.global_cfg.variables,
)
> verify_tests(self.spec)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/pytest/item.py:194:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test_spec = {'test_name': 'Test accept valid fleet provision request', 'includes': [{'name': 'device test data valid', 'descriptio...emplate/provision/json/accepted', 'json': {'deviceConfiguration': {}, 'thingName': '{deviceUuid:s}'}, 'timeout': 10}}]}
with_plugins = True
def verify_tests(test_spec, with_plugins=True):
"""Verify that a specific test block is correct
Todo:
Load schema file once. Requires some caching of the file
Args:
test_spec (dict): Test in dictionary form
Raises:
BadSchemaError: Schema did not match
"""
here = os.path.dirname(os.path.abspath(__file__))
schema_filename = os.path.join(here, "tests.jsonschema.yaml")
schema = load_schema_file(schema_filename, with_plugins)
> verify_jsonschema(test_spec, schema)
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/schema/files.py:148:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
to_verify = {'test_name': 'Test accept valid fleet provision request', 'includes': [{'name': 'device test data valid', 'descriptio...emplate/provision/json/accepted', 'json': {'deviceConfiguration': {}, 'thingName': '{deviceUuid:s}'}, 'timeout': 10}}]}
schema = {'$schema': 'http://json-schema.org/draft-07/schema#', '$id': 'https://raw.githubusercontent.com/taverntesting/tavern/...': False, 'required': ['username'], 'properties': {'username': {'type': 'string'}, 'password': {'type': 'string'}}}}}}}
def verify_jsonschema(to_verify, schema):
"""Verify a generic file against a given jsonschema
Args:
to_verify (dict): Filename of source tests to check
schema (dict): Schema to verify against
Raises:
BadSchemaError: Schema did not match
"""
# pylint: disable=too-many-locals
validator = CustomValidator(schema)
try:
validator.validate(to_verify)
except jsonschema.ValidationError as e:
real_context = []
# ignore these strings because they're red herrings
for c in e.context:
description = c.schema.get("description", "<no description>")
if description == "Reference to another stage from an included config file":
continue
instance = c.instance
filename = get_stage_filename(instance)
if filename is None:
# Depending on what block raised the error, it mightbe difficult to tell what it was, so check the parent too
instance = e.instance
filename = get_stage_filename(instance)
if filename:
with open(filename, "r", encoding="utf-8") as infile:
n_lines = len(infile.readlines())
first_line, last_line, _ = get_stage_lines(instance)
first_line = max(first_line - 2, 0)
last_line = min(last_line + 2, n_lines)
reg = re.compile(r"^\s*$")
lines = read_relevant_lines(instance, first_line, last_line)
lines = [line for line in lines if not reg.match(line.strip())]
content = "\n".join(list(lines))
real_context.append(
f"""
{c.message}
{filename}: line {first_line}-{last_line}:
{content}
"""
)
else:
real_context.append(
f"""
{c.message}
<error: unable to find input file for context>
"""
)
msg = "\n---\n" + "\n---\n".join([str(i) for i in real_context])
> raise BadSchemaError(msg) from None
E tavern._core.exceptions.BadSchemaError:
E ---
E
E {'topic': '$aws/certificates/create/json/accepted', 'json': {'certificateId': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'certificateOwnershipToken': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'certificatePem': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>, 'privateKey': <tavern._core.loader.AnythingSentinel object at 0x10646d8b0>}, 'verify_response_with': {'function': 'save_certificate:save_certificate'}, 'save': {'json': {'createCertResp_certificateId': 'certificateId', 'createCertResp_certificatePem': 'certificatePem', 'createCertResp_privateKey': 'privateKey', 'createCertResp_certificateOwnershipToken': 'certificateOwnershipToken'}}, 'timeout': 10} is not valid under any of the given schemas
E /Users/alexberger/dev/aai/cloud-infrastructure/integration-tests/fleet-provisioning/test_accept_valid_fleet_provision.tavern.yml: line 23-44:
E
E json: {}
E mqtt_response:
E topic: $aws/certificates/create/json/accepted
E json:
E certificateId: !anything
E certificateOwnershipToken: !anything
E certificatePem: !anything
E privateKey: !anything
E verify_response_with:
E function: save_certificate:save_certificate
E save:
E json:
E createCertResp_certificateId: certificateId
E createCertResp_certificatePem: certificatePem
E createCertResp_privateKey: privateKey
E createCertResp_certificateOwnershipToken: certificateOwnershipToken
E timeout: 10
E - name: Provision Certificate
E mqtt_publish:
../../../../../.pyenv/versions/fp/lib/python3.9/site-packages/tavern/_core/schema/jsonschema.py:171: BadSchemaError
michaelboulton commented
Can you give an example of the test you were running to get the error?
michaelboulton commented
I'm guessing this was due to the accidental breaking changes to 'save' in MQTT blocks, this should be fixed in 2.0.2