Failing unit tests for stdlib on Python3.9
Closed this issue · 3 comments
pirat89 commented
================================================================================================================== FAILURES ==================================================================================================================
_____________________________________________________________________________________________________________ test_stdin_string ______________________________________________________________________________________________________________
def test_stdin_string():
> ret = run(('bash', '-c', 'read MSG; echo "<$MSG>"'), stdin='LOREM IPSUM')
tests/scripts/test_stdlib.py:43:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
leapp/libraries/stdlib/__init__.py:180: in run
result = _call(args, callback_raw=callback_raw, callback_linebuffered=callback_linebuffered,
leapp/libraries/stdlib/call.py:171: in _call
read = _multiplex(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ep = <select.epoll object at 0x7ff27b759a10>, read_fds = [22, 25], callback_raw = <function _console_logging_handler at 0x7ff27d2f41f0>, callback_linebuffered = <function _logfile_logging_handler at 0x7ff27d2f4280>, encoding = 'utf-8'
write = (28, b'LOREM IPSUM'), timeout = 1, buffer_size = 80
def _multiplex(ep, read_fds, callback_raw, callback_linebuffered,
encoding='utf-8', write=None, timeout=1, buffer_size=80):
# Register the file descriptors (stdout + stderr) with the epoll object
# so that we'll get notifications when data are ready to read
for fd in read_fds:
ep.register(fd, POLL_IN | POLL_PRI)
# Register a write file descriptor
if write:
ep.register(write[0], POLL_OUT)
# Offset into the `write[1]` buffer where we should continue writing to stdin
offset = 0
# We need to keep track of which file descriptors have already been drained
# because when running under `pytest` it seems that all `epoll` events are
# received twice so using solely `ep.unregister(fd)` will not work
hupped = set()
# Total number of 'hupped' file descriptors we expect
num_expected = len(read_fds) + (1 if write else 0)
# Set up file-descriptor specific buffers where we'll buffer the output
buf = {fd: bytes() for fd in read_fds}
if encoding:
linebufs = {fd: '' for fd in read_fds}
decoders = {fd: codecs.getincrementaldecoder(encoding)() for fd in read_fds}
def _get_fd_type(fd):
"""
File descriptors passed via `read_fds` are always representing [stdout, stderr],
since arrays start at index 0, we need to add 1 to get the real symbolic value
`STDOUT` or `STDERR`.
"""
return read_fds.index(fd) + 1
while not ep.closed and len(hupped) != num_expected:
events = ep.poll(timeout)
for fd, event in events:
if event == POLL_HUP:
hupped.add(fd)
ep.unregister(fd)
if event & (POLL_IN | POLL_PRI) != 0:
fd_type = _get_fd_type(fd)
read = os.read(fd, buffer_size)
callback_raw((fd, fd_type), read)
if encoding:
linebufs[fd] += decoders[fd].decode(read)
while '\n' in linebufs[fd]:
pre, post = linebufs[fd].split('\n', 1)
linebufs[fd] = post
callback_linebuffered((fd, fd_type), pre)
buf[fd] += read
elif event == POLL_OUT:
# Write data to pipe, `os.write` returns the number of bytes written,
# thus we need to offset
wfd, data = write
if fd in hupped:
continue
offset += os.write(fd, data[offset:])
if offset == len(data):
os.close(fd)
hupped.add(fd)
> ep.unregister(fd)
E OSError: [Errno 9] Bad file descriptor
leapp/libraries/stdlib/call.py:75: OSError
_____________________________________________________________________________________________________________ test_stdin_string ______________________________________________________________________________________________________________
def test_stdin_string():
> ret = _call(('bash', '-c', 'read MSG; echo "<$MSG>"'), stdin='LOREM IPSUM')
tests/scripts/test_stdlib_call.py:19:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
leapp/libraries/stdlib/call.py:171: in _call
read = _multiplex(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
ep = <select.epoll object at 0x7ff27b759d50>, read_fds = [44, 46], callback_raw = <function <lambda> at 0x7ff27d2ef700>, callback_linebuffered = <function <lambda> at 0x7ff27d2efc10>, encoding = 'utf-8', write = (49, b'LOREM IPSUM')
timeout = 1, buffer_size = 80
def _multiplex(ep, read_fds, callback_raw, callback_linebuffered,
encoding='utf-8', write=None, timeout=1, buffer_size=80):
# Register the file descriptors (stdout + stderr) with the epoll object
# so that we'll get notifications when data are ready to read
for fd in read_fds:
ep.register(fd, POLL_IN | POLL_PRI)
# Register a write file descriptor
if write:
ep.register(write[0], POLL_OUT)
# Offset into the `write[1]` buffer where we should continue writing to stdin
offset = 0
# We need to keep track of which file descriptors have already been drained
# because when running under `pytest` it seems that all `epoll` events are
# received twice so using solely `ep.unregister(fd)` will not work
hupped = set()
# Total number of 'hupped' file descriptors we expect
num_expected = len(read_fds) + (1 if write else 0)
# Set up file-descriptor specific buffers where we'll buffer the output
buf = {fd: bytes() for fd in read_fds}
if encoding:
linebufs = {fd: '' for fd in read_fds}
decoders = {fd: codecs.getincrementaldecoder(encoding)() for fd in read_fds}
def _get_fd_type(fd):
"""
File descriptors passed via `read_fds` are always representing [stdout, stderr],
since arrays start at index 0, we need to add 1 to get the real symbolic value
`STDOUT` or `STDERR`.
"""
return read_fds.index(fd) + 1
while not ep.closed and len(hupped) != num_expected:
events = ep.poll(timeout)
for fd, event in events:
if event == POLL_HUP:
hupped.add(fd)
ep.unregister(fd)
if event & (POLL_IN | POLL_PRI) != 0:
fd_type = _get_fd_type(fd)
read = os.read(fd, buffer_size)
callback_raw((fd, fd_type), read)
if encoding:
linebufs[fd] += decoders[fd].decode(read)
while '\n' in linebufs[fd]:
pre, post = linebufs[fd].split('\n', 1)
linebufs[fd] = post
callback_linebuffered((fd, fd_type), pre)
buf[fd] += read
elif event == POLL_OUT:
# Write data to pipe, `os.write` returns the number of bytes written,
# thus we need to offset
wfd, data = write
if fd in hupped:
continue
offset += os.write(fd, data[offset:])
if offset == len(data):
os.close(fd)
hupped.add(fd)
> ep.unregister(fd)
E OSError: [Errno 9] Bad file descriptor
leapp/libraries/stdlib/call.py:75: OSError
_______________________________________________________________________________________________________________ test_output_1 ________________________________________________________________________________________________________________
def test_output_1():
ret = _call(('false',))
assert isinstance(ret['exit_code'], int)
> assert ret['exit_code'] == 1
E assert 0 == 1
tests/scripts/test_stdlib_call.py:52: AssertionError
_______________________________________________________________________________________________________________ test_output_2 ________________________________________________________________________________________________________________
def test_output_2():
ret = _call(('bash', '-c', 'echo STDOUT; (exec >&2 ; echo STDERR); exit 42',))
assert isinstance(ret['exit_code'], int)
> assert ret['exit_code'] == 42
E assert 0 == 42
tests/scripts/test_stdlib_call.py:64: AssertionError
_____________________________________________________________________________________________________________ test_env_injection _____________________________________________________________________________________________________________
def test_env_injection():
ret = _call(('bash', '-c', 'echo $TEST'), env={'TEST': 'SUCCESS'})
assert isinstance(ret['exit_code'], int)
> assert ret['exit_code'] == 0
E assert 1 == 0
tests/scripts/test_stdlib_call.py:76: AssertionError
----------- coverage: platform linux, python 3.9.1-final-0 -----------
Name
Additional info:
- executed on Fedora33
shaded-enmity commented
@pirat89 can you please try the following patch? Looks like something changed how epoll unregisters file descriptors.
diff --git a/leapp/libraries/stdlib/call.py b/leapp/libraries/stdlib/call.py
index 9ecb70e..4f16ea3 100644
--- a/leapp/libraries/stdlib/call.py
+++ b/leapp/libraries/stdlib/call.py
@@ -70,9 +70,9 @@ def _multiplex(ep, read_fds, callback_raw, callback_linebuffered,
continue
offset += os.write(fd, data[offset:])
if offset == len(data):
- os.close(fd)
hupped.add(fd)
ep.unregister(fd)
+ os.close(fd)
# Process leftovers from line buffering
if encoding:pirat89 commented
@shaded-enmity yes. this fixes the issue.
pirat89 commented
Here is the reason: python/cpython@5b23f76
And: https://bugs.python.org/issue39239
So this is really expected fix.