JuliaHubOSS/llvm-cbe

Investigate flakiness in "char sized pointer" tests

dpaoliello opened this issue · 0 comments

test_char_sized_ptr_math_decr and test_char_sized_ptr_math_incr are flaky on Windows, for example: https://github.com/JuliaHubOSS/llvm-cbe/actions/runs/5594323520/jobs/10229051315

4294967046==0xFFFFFF06 which suggests there some weirdness going on with overflow/underflow.

______ test_consistent_return_value_c[test_char_sized_ptr_math_decr--O1] ______
[gw8] win32 -- Python 3.11.4 C:\hostedtoolcache\windows\Python\3.11.4\x64\python.exe

test_filename = 'D:\\a\\llvm-cbe\\llvm-cbe\\llvm\\projects\\llvm-cbe\\test\\c_tests\\pointers\\test_char_sized_ptr_math_decr.c'
tmpdir = local('C:\\Users\\runneradmin\\AppData\\Local\\Temp\\pytest-of-runneradmin\\pytest-0\\popen-gw8\\test_consistent_return_value_c22')
cflags = ['-O1']

    @pytest.mark.parametrize(
        'cflags',
        [['-O0'], ['-O1'], ['-O2'], ['-O3']],
        ids=lambda flags: ' '.join(flags)
    )
    @pytest.mark.parametrize(
        'test_filename',
        collect_tests(TEST_DIR, ('.c', '.cpp')),
        ids=get_test_name_from_filename,
    )
    def test_consistent_return_value_c(test_filename, tmpdir, cflags):
        """
        Compile and execute a C or C++ file with clang, and compare its exit code
        with the exit code when compiled with llvm-cbe followed by gcc.
    
        Also, the exit code must be TEST_SUCCESS_EXIT_CODE for success or
        TEST_XFAIL_EXIT_CODE or expected failures.
        """
    
        check_xfail(test_filename)
    
        cplusplus = test_filename.endswith('.cpp')
    
        # make sure CBE doesn't have any errors before trying to compile
        # executables
        ir = compile_to_ir(
            test_filename, tmpdir / 'ir.ll', flags=cflags, cplusplus=cplusplus)
        cbe_c = run_llvm_cbe(ir, tmpdir / 'cbe.c')
    
        regular_exe = compile_clang(
            test_filename,
            tmpdir / 'regular.exe',
            flags=cflags,
            cplusplus=cplusplus)
        regular_retval = call([regular_exe])
        print('regular executable returned', regular_retval)
        assert regular_retval in [TEST_SUCCESS_EXIT_CODE, TEST_XFAIL_EXIT_CODE]
    
        # suppress "array subscript -1 is outside array bounds" in test expected
        # to trigger it
        if not USE_MSVC and "test_char_sized_ptr_math_decr" in test_filename:
            # not += to avoid affecting subsequent calls
            cflags = cflags + ["-Wno-array-bounds"]
    
        if USE_MSVC:
            map_flags = {'-O3': '-O2', '-O0': '-Od'}
            cbe_exe = compile_msvc(cbe_c, tmpdir / 'cbe.exe', flags=[f if f not in map_flags else map_flags[f] for f in cflags])
        else:
            cbe_exe = compile_gcc(cbe_c, tmpdir / 'cbe.exe', flags=cflags)
        cbe_retval = call([cbe_exe])
        print('cbe output returned', cbe_retval)
>       assert cbe_retval == regular_retval
E       assert 4294967046 == 6

test\test_cbe.py:205: AssertionError
---------------------------- Captured stdout call -----------------------------
regular executable returned 6
cbe output returned 4294967046