k8s_info openshift.dynamic.exceptions.ServiceUnavailableError: 503
wmlynch opened this issue · 3 comments
wmlynch commented
Target cluster version: Openshift 4.2.5
openshift pip version: 0.10.0
kubernetes pip version: 10.0.1
ansible 2.9.2
- name: wait for kubernetes service
k8s_info:
api_version: v1
kind: Service
kubeconfig: "{{ cluster_kubeconfig }}"
name: kubernetes
namespace: default
register: _kube_service
Ansible std_error block:
"stderr_lines": [
"Traceback (most recent call last):",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 42, in inner",
" resp = func(self, *args, **kwargs)",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 241, in request",
" _return_http_data_only=params.get('_return_http_data_only', True)",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 334, in call_api",
" _return_http_data_only, collection_formats, _preload_content, _request_timeout)",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 168, in __call_api",
" _request_timeout=_request_timeout)",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 355, in request",
" headers=headers)",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 231, in GET",
" query_params=query_params)",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 222, in request",
" raise ApiException(http_resp=r)",
"kubernetes.client.rest.ApiException: (503)",
"Reason: Service Unavailable",
"HTTP response headers: HTTPHeaderDict({'Cache-Control': 'no-cache, private', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Date': 'Thu, 12 Dec 2019 23:00:40 GMT', 'Content-Length': '20'})",
"HTTP response body: b'service unavailable\\n'",
"",
"",
"During handling of the above exception, another exception occurred:",
"",
"Traceback (most recent call last):",
" File \"/root/.ansible/tmp/ansible-tmp-1576191621.8594584-53701596979993/AnsiballZ_k8s_info.py\", line 102, in <module>",
" _ansiballz_main()",
" File \"/root/.ansible/tmp/ansible-tmp-1576191621.8594584-53701596979993/AnsiballZ_k8s_info.py\", line 94, in _ansiballz_main",
" invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)",
" File \"/root/.ansible/tmp/ansible-tmp-1576191621.8594584-53701596979993/AnsiballZ_k8s_info.py\", line 40, in invoke_module",
" runpy.run_module(mod_name='ansible.modules.clustering.k8s.k8s_info', init_globals=None, run_name='__main__', alter_sys=True)",
" File \"/usr/lib/python3.7/runpy.py\", line 205, in run_module",
" return _run_module_code(code, init_globals, run_name, mod_spec)",
" File \"/usr/lib/python3.7/runpy.py\", line 96, in _run_module_code",
" mod_name, mod_spec, pkg_name, script_name)",
" File \"/usr/lib/python3.7/runpy.py\", line 85, in _run_code",
" exec(code, run_globals)",
" File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/modules/clustering/k8s/k8s_info.py\", line 179, in <module>",
" File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/modules/clustering/k8s/k8s_info.py\", line 175, in main",
" File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/modules/clustering/k8s/k8s_info.py\", line 156, in execute_module",
" File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/module_utils/k8s/common.py\", line 215, in kubernetes_facts",
" File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/module_utils/k8s/common.py\", line 205, in find_resource",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 201, in get",
" results = self.search(**kwargs)",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 242, in search",
" results = self.__search(self.__build_search(**kwargs), self.__resources, [])",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 284, in __search",
" matches.extend(self.__search([key] + parts[1:], resources, reqParams))",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 270, in __search",
" return self.__search(parts[1:], resourcePart, reqParams + [part] )",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 284, in __search",
" matches.extend(self.__search([key] + parts[1:], resources, reqParams))",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 270, in __search",
" return self.__search(parts[1:], resourcePart, reqParams + [part] )",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 263, in __search",
" group, part, resourcePart.preferred)",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 166, in get_resources_for_api_version",
" resources_response = self.client.request('GET', path).resources or []",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 44, in inner",
" raise api_exception(e)",
"openshift.dynamic.exceptions.ServiceUnavailableError: 503",
"Reason: Service Unavailable",
"HTTP response headers: HTTPHeaderDict({'Cache-Control': 'no-cache, private', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Date': 'Thu, 12 Dec 2019 23:00:40 GMT', 'Content-Length': '20'})",
"HTTP response body: b'service unavailable\\n'",
"Original traceback: ",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 42, in inner",
" resp = func(self, *args, **kwargs)",
"",
" File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 241, in request",
" _return_http_data_only=params.get('_return_http_data_only', True)",
"",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 334, in call_api",
" _return_http_data_only, collection_formats, _preload_content, _request_timeout)",
"",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 168, in __call_api",
" _request_timeout=_request_timeout)",
"",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 355, in request",
" headers=headers)",
"",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 231, in GET",
" query_params=query_params)",
"",
" File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 222, in request",
" raise ApiException(http_resp=r)",
""
]
Inspecting the cluster, post k8s_info failure, the following apiservices are in a failed state.
apiservice.apiregistration.k8s.io/v1.packages.operators.coreos.com openshift-operator-lifecycle-manager/v1-packages-operators-coreos-com False (FailedDiscoveryCheck)
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io openshift-monitoring/prometheus-adapter False (FailedDiscoveryCheck)
I'm surmising that one or both of the apiservices in FailedDiscoveryCheck caused the failure. What I'm struggling to understand is why.
Some questions/observations:
- Based on discussions in issue #327 the openshift client is gathering all
v1resources, Is this correct? - If so, why?
- I suspect that if I called
oc get service -n default kubernetes -o yamlthe command would work successfully.
fabianvf commented
Hmm, I'm not sure why it would be trying to hit the endpoints that are down, unless discovery as a whole is broken because of it. It should be lazily accessing endpoints as needed,
Some questions/observations:
- Based on discussions in issue #327 the openshift client is gathering all v1 resources, Is this correct?
Yes, we're discovering all the core V1 resources (ie, /api/v1)
- If so, why?
The client is fully dynamic, and uses the discovery interface to discover all resources, including the core resources. This lets us handle it all very generically without needing any special casing for core resources.
- I suspect that if I called oc get service -n default kubernetes -o yaml the command would work successfully.
Yes I'd expect that's true, though could you try?
uditgaurav commented
I am also facing a similar problem with
openshift version: 4.2.0
python version is 2.7.12
ansible version is 2.8.0
Error is reporting:
The full traceback is:
Traceback (most recent call last):
File "/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py", line 114, in <module>
_ansiballz_main()
File "/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py", line 106, in _ansiballz_main
invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)
File "/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py", line 49, in invoke_module
imp.load_module('__main__', mod, module, MOD_DESC)
File "/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py", line 176, in <module>
File "/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py", line 172, in main
File "/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py", line 153, in execute_module
File "/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py", line 199, in kubernetes_facts
File "/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py", line 189, in find_resource
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 227, in get
results = self.search(**kwargs)
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 272, in search
results = self.__search(self.__build_search(**kwargs), self.__resources, [])
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 314, in __search
matches.extend(self.__search([key] + parts[1:], resources, reqParams))
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 300, in __search
return self.__search(parts[1:], resourcePart, reqParams + [part] )
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 314, in __search
matches.extend(self.__search([key] + parts[1:], resources, reqParams))
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 300, in __search
return self.__search(parts[1:], resourcePart, reqParams + [part] )
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 293, in __search
group, part, resourcePart.preferred)
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 185, in get_resources_for_api_version
resources_response = self.client.request('GET', path).resources or []
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py", line 44, in inner
raise api_exception(e)
openshift.dynamic.exceptions.ServiceUnavailableError: 503
Reason: Service Unavailable
HTTP response headers: HTTPHeaderDict({'Date': 'Fri, 27 Dec 2019 12:27:35 GMT', 'Content-Length': '20', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-cache, private'})
HTTP response body: service unavailable
Original traceback:
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py", line 42, in inner
resp = func(self, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py", line 245, in request
_return_http_data_only=params.get('_return_http_data_only', True)
File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py", line 334, in call_api
_return_http_data_only, collection_formats, _preload_content, _request_timeout)
File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py", line 168, in __call_api
_request_timeout=_request_timeout)
File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py", line 355, in request
headers=headers)
File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py", line 231, in GET
query_params=query_params)
File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py", line 222, in request
raise ApiException(http_resp=r)
fatal: [localhost]: FAILED! => {
"changed": false,
"module_stderr": "Traceback (most recent call last):\n File \"/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py\", line 114, in <module>\n _ansiballz_main()\n File \"/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py\", line 106, in _ansiballz_main\n invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)\n File \"/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py\", line 49, in invoke_module\n imp.load_module('__main__', mod, module, MOD_DESC)\n File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py\", line 176, in <module>\n File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py\", line 172, in main\n File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py\", line 153, in execute_module\n File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py\", line 199, in kubernetes_facts\n File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py\", line 189, in find_resource\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 227, in get\n results = self.search(**kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 272, in search\n results = self.__search(self.__build_search(**kwargs), self.__resources, [])\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 314, in __search\n matches.extend(self.__search([key] + parts[1:], resources, reqParams))\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 300, in __search\n return self.__search(parts[1:], resourcePart, reqParams + [part] )\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 314, in __search\n matches.extend(self.__search([key] + parts[1:], resources, reqParams))\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 300, in __search\n return self.__search(parts[1:], resourcePart, reqParams + [part] )\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 293, in __search\n group, part, resourcePart.preferred)\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 185, in get_resources_for_api_version\n resources_response = self.client.request('GET', path).resources or []\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py\", line 44, in inner\n raise api_exception(e)\nopenshift.dynamic.exceptions.ServiceUnavailableError: 503\nReason: Service Unavailable\nHTTP response headers: HTTPHeaderDict({'Date': 'Fri, 27 Dec 2019 12:27:35 GMT', 'Content-Length': '20', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-cache, private'})\nHTTP response body: service unavailable\n\nOriginal traceback: \n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py\", line 42, in inner\n resp = func(self, *args, **kwargs)\n\n File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py\", line 245, in request\n _return_http_data_only=params.get('_return_http_data_only', True)\n\n File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py\", line 334, in call_api\n _return_http_data_only, collection_formats, _preload_content, _request_timeout)\n\n File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py\", line 168, in __call_api\n _request_timeout=_request_timeout)\n\n File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py\", line 355, in request\n headers=headers)\n\n File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py\", line 231, in GET\n query_params=query_params)\n\n File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py\", line 222, in request\n raise ApiException(http_resp=r)\n\n",
"module_stdout": "",
"msg": "MODULE FAILURE\nSee stdout/stderr for the exact error",
"rc": 1
}
To reproduce run below ansible in above version
- hosts: localhost
tasks:
- name: Get a list of all pods
k8s_facts:
api_version: v1
kind: Pod
namespace: default
register: pod_list
Is there any solution or workaround to resolve the above issue?
wmlynch commented
@uditgaurav This PR fixes the issue for me: #350