Even no extra-inventory is specified, extra_inventory_manager will still be initialized.
nhe-NV opened this issue · 2 comments
nhe-NV commented
In the plugin.py, the option --extra-inventory is defined as following, the extra-inventory will always be there.
"extra_inventory" in self.options will always be true.
This cause even no extra-inventory is specified, the extra_inventory_manager will defined, and the it will cause the ansible command run twice.
group.addoption(
"--extra-inventory",
"--ansible-extra-inventory",
action="store",
dest="ansible_extra_inventory",
default=None,
metavar="ANSIBLE_EXTRA_INVENTORY",
help="ansible extra inventory file URI (default: %(default)s)",
)
class HostManagerV29(BaseHostManager):
"""Fixme."""
def __init__(self, *args, **kwargs) -> None:
"""Fixme."""
super().__init__(*args, **kwargs)
self._dispatcher = ModuleDispatcherV29
def initialize_inventory(self):
self.options["loader"] = DataLoader()
self.options["inventory_manager"] = InventoryManager(
loader=self.options["loader"],
sources=self.options["inventory"],
)
self.options["variable_manager"] = VariableManager(
loader=self.options["loader"],
inventory=self.options["inventory_manager"],
)
**if "extra_inventory" in self.options:**
self.options["extra_loader"] = DataLoader()
self.options["extra_inventory_manager"] = InventoryManager(
loader=self.options["extra_loader"],
sources=self.options["extra_inventory"],
)
self.options["extra_variable_manager"] = VariableManager(
loader=self.options["extra_loader"],
inventory=self.options["extra_inventory_manager"],
)
class ModuleDispatcherV29(ModuleDispatcherV2):
......
if "extra_inventory_manager" in self.options:
tqm_extra = None
try:
tqm_extra = TaskQueueManager(**kwargs_extra)
tqm_extra.run(play_extra)
finally:
if tqm_extra:
tqm_extra.cleanup()
wangxin commented
We also noticed ansible module executed twice issue.
Sampe pytest script to reproduce this issue:
import logging
import json
logger = logging.getLogger(__name__)
def test_case1(ansible_adhoc):
hosts = ansible_adhoc()
res = hosts['localhost'].shell("rm aaa.txt")
logger.debug("Result: {}".format(json.dumps(res.contacted)))
res = hosts['localhost'].shell("echo a >> aaa.txt")
logger.debug("Result: {}".format(json.dumps(res.contacted)))
res = hosts['localhost'].shell("cat aaa.txt")
logger.debug("Result: {}".format(json.dumps(res.contacted, indent=4)))Log of running this sample script (With some extra log message printed by debug code added by me). Although shell command "echo a >> aaa.txt" is called only once, the created "aaa.txt" file always has 2 lines.
xiwang5@sonic-mgmt-xiwang5:~/code/duprun$ python3 -m pytest test_module.py --inventory inv --host-pattern all --log-cli-level debug -vvv --pdb
Ansible version: 2.13.11
has_version_v29: True
has_version_v212: True
has_version_v213: True
INFO:pytest_ansible.units:Looking for collection info in /home/xiwang5/code/duprun/galaxy.yml
ERROR:pytest_ansible.units:No galaxy.yml file found, plugin not activated
===================================================================================================================== test session starts ======================================================================================================================
platform linux -- Python 3.8.10, pytest-7.1.3, pluggy-1.2.0 -- /usr/bin/python3
cachedir: .pytest_cache
metadata: {'Python': '3.8.10', 'Platform': 'Linux-5.19.0-45-generic-x86_64-with-glibc2.29', 'Packages': {'pytest': '7.1.3', 'pluggy': '1.2.0'}, 'Plugins': {'repeat': '0.9.1', 'metadata': '3.0.0', 'xdist': '1.28.0', 'html': '3.2.0', 'allure-pytest': '2.8.22
', 'forked': '1.6.0', 'ansible': '4.0.0'}}
ansible: 2.13.11
rootdir: /home/xiwang5/code/duprun
plugins: repeat-0.9.1, metadata-3.0.0, xdist-1.28.0, html-3.2.0, allure-pytest-2.8.22, forked-1.6.0, ansible-4.0.0
collected 1 item
test_module.py::test_case1
------------------------------------------------------------------------------------------------------------------------ live log call -------------------------------------------------------------------------------------------------------------------------
INFO pytest_ansible.plugin:plugin.py:399 ansible_cfg: {'inventory': 'inv', 'extra_inventory': None, 'host_pattern': 'all', 'connection': 'smart', 'user': None, 'module_path': ['/home/xiwang5/code/sonic-mgmt2/ansible'], 'become': False, 'become_method':
'sudo', 'become_user': 'root', 'ask_become_pass': False, 'subset': None}
INFO pytest_ansible.plugin:plugin.py:403 ansible_cfg with request: {'inventory': 'inv', 'extra_inventory': None, 'host_pattern': 'all', 'connection': 'smart', 'user': None, 'module_path': ['/home/xiwang5/code/sonic-mgmt2/ansible'], 'become': False, 'be
come_method': 'sudo', 'become_user': 'root', 'ask_become_pass': False, 'subset': None}
INFO duprun:v213.py:25 ModuleDispatcher File: /usr/local/lib/python3.8/dist-packages/pytest_ansible/module_dispatcher/v213.py
INFO duprun:v213.py:78 Trying to load module shell
INFO duprun:v213.py:91 Trying to run module with module_args=('rm aaa.txt',), complex_args={}
INFO duprun:v213.py:198 play_ds: {'name': 'pytest-ansible', 'hosts': 'localhost', 'become': False, 'become_user': 'root', 'gather_facts': 'no', 'tasks': [{'action': {'module': 'shell', 'args': {'_raw_params': 'rm aaa.txt'}}}]}
WARNING duprun:task_queue_manager.py:257 In ansible tqm run
WARNING duprun:task_queue_manager.py:257 In ansible tqm run
DEBUG test_module:test_module.py:11 Result: {"localhost": {"failed": true, "changed": true, "stdout": "", "stderr": "rm: cannot remove 'aaa.txt': No such file or directory", "rc": 1, "cmd": "rm aaa.txt", "start": "2023-08-16 04:56:55.101583", "end": "20
23-08-16 04:56:55.104060", "delta": "0:00:00.002477", "msg": "non-zero return code", "invocation": {"module_args": {"_raw_params": "rm aaa.txt", "_uses_shell": true, "warn": false, "stdin_add_newline": true, "strip_empty_ends": true, "argv": null, "chdir":
null, "executable": null, "creates": null, "removes": null, "stdin": null}}, "stdout_lines": [], "stderr_lines": ["rm: cannot remove 'aaa.txt': No such file or directory"], "_ansible_no_log": null}}
INFO duprun:v213.py:78 Trying to load module shell
INFO duprun:v213.py:91 Trying to run module with module_args=('echo a >> aaa.txt',), complex_args={}
INFO duprun:v213.py:198 play_ds: {'name': 'pytest-ansible', 'hosts': 'localhost', 'become': False, 'become_user': 'root', 'gather_facts': 'no', 'tasks': [{'action': {'module': 'shell', 'args': {'_raw_params': 'echo a >> aaa.txt'}}}]}
WARNING duprun:task_queue_manager.py:257 In ansible tqm run
WARNING duprun:task_queue_manager.py:257 In ansible tqm run
DEBUG test_module:test_module.py:14 Result: {"localhost": {"changed": true, "stdout": "", "stderr": "", "rc": 0, "cmd": "echo a >> aaa.txt", "start": "2023-08-16 04:56:55.370810", "end": "2023-08-16 04:56:55.372663", "delta": "0:00:00.001853", "msg": ""
, "invocation": {"module_args": {"_raw_params": "echo a >> aaa.txt", "_uses_shell": true, "warn": false, "stdin_add_newline": true, "strip_empty_ends": true, "argv": null, "chdir": null, "executable": null, "creates": null, "removes": null, "stdin": null}}
, "stdout_lines": [], "stderr_lines": [], "_ansible_no_log": null}}
INFO duprun:v213.py:78 Trying to load module shell
INFO duprun:v213.py:91 Trying to run module with module_args=('cat aaa.txt',), complex_args={}
INFO duprun:v213.py:198 play_ds: {'name': 'pytest-ansible', 'hosts': 'localhost', 'become': False, 'become_user': 'root', 'gather_facts': 'no', 'tasks': [{'action': {'module': 'shell', 'args': {'_raw_params': 'cat aaa.txt'}}}]}
WARNING duprun:task_queue_manager.py:257 In ansible tqm run
WARNING duprun:task_queue_manager.py:257 In ansible tqm run
DEBUG test_module:test_module.py:18 Result: {
"localhost": {
"changed": true,
"stdout": "a\na",
"stderr": "",
"rc": 0,
"cmd": "cat aaa.txt",
"start": "2023-08-16 04:56:55.634639",
"end": "2023-08-16 04:56:55.637056",
"delta": "0:00:00.002417",
"msg": "",
"invocation": {
"module_args": {
"_raw_params": "cat aaa.txt",
"_uses_shell": true,
"warn": false,
"stdin_add_newline": true,
"strip_empty_ends": true,
"argv": null,
"chdir": null,
"executable": null,
"creates": null,
"removes": null,
"stdin": null
}
},
"stdout_lines": [
"a",
"a"
],
"stderr_lines": [],
"_ansible_no_log": null
}
}
PASSED [100%]
======================================================================================================================= warnings summary =======================================================================================================================
test_module.py::test_case1
test_module.py::test_case1
/usr/local/lib/python3.8/dist-packages/pytest_ansible/module_dispatcher/v213.py:105: UserWarning: provided hosts list is empty, only localhost is available
warnings.warn("provided hosts list is empty, only localhost is available")
test_module.py::test_case1
test_module.py::test_case1
test_module.py::test_case1
test_module.py::test_case1
test_module.py::test_case1
test_module.py::test_case1
/usr/local/lib/python3.8/dist-packages/ansible/executor/task_queue_manager.py:257: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead
logger.warn("In ansible tqm run")
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
================================================================================================================ 1 passed, 8 warnings in 1.12s =================================================================================================================
wangxin commented
IMO, there are 2 issues under the hood.
- The logic for deciding whether to create extra_inventory_manager is not robust as @nhe-NV indicated. In
host_manager/v2xx.py, the original code is like below:
if "extra_inventory" in self.options:
This only can tell whether self.options has key extra_inventory. However value of self.options["extra_inventory"] could be None. More robust code would be like below:
if self.options.get("extra_inventory", None):
- When extra_inventory is specified and extra_inventory_manager is indeed created, ansible module could be executed twice for hosts in both inventories. For example
localhost.