harvest: add a test case for ffmpeg timeout

Question

harvest: add a test case for ffmpeg timeout

Opened this issue 2 years ago · 0 comments

It seems in some cases, when ffmpeg times out, the harvester will attempt to retry the jobs, but eventually get stuck on the :new status and not progress past the extraction phase

Error output from an example job:

{"error":"\nExternal Program: status=;killed=true;pid=19973;time_out_sec=60.0;time_taken_sec=61.27372472197749;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=23317;time_out_sec=60.0;time_taken_sec=61.27367147500627;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=22938;time_out_sec=60.0;time_taken_sec=61.28113416500855;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=22115;time_out_sec=60.0;time_taken_sec=61.28208795504179;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=19794;time_out_sec=60.0;time_taken_sec=61.26551218307577;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=22986;time_out_sec=60.0;time_taken_sec=61.269231615937315;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=24335;time_out_sec=60.0;time_taken_sec=61.256726833991706;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=24472;time_out_sec=60.0;time_taken_sec=61.26816164003685;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=25891;time_out_sec=60.0;time_taken_sec=61.26841467095073;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=22482;time_out_sec=60.0;time_taken_sec=61.26929898210801;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=21102;time_out_sec=60.0;time_taken_sec=61.27420623507351;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n\nExternal Program: status=;killed=true;pid=21613;time_out_sec=60.0;time_taken_sec=61.26980184693821;timed_out=true;exceptions=[];command=ffmpeg -hide_banner -loglevel repeat+verbose -nostdin -i \"/data/harvester_to_do/harvest_12/Toorale-National-Park-Dry-B_323/267/20210105_-30.2647+145.3586/data\" -codec copy -f null -\n\tStandard output: \n\tStandard Error: \n\n","fixes":[{"version":1,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":2,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":3,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":4,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":5,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":6,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":7,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":8,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":9,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":10,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":11,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}},{"version":12,"problems":{"FL001":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL005":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"FL008":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}},"FL010":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"Audio recording is not a FLAC file","severity":"None"}},"FL011":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Error","message":"Audio recording is not a FLAC file","severity":"None"}},"FL012":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":null,"severity":"None"}},"WA002":{"status":"NoOperation","message":null,"new_path":null,"check_result":{"data":null,"status":"Unaffected","message":"","severity":"None"}}}}],"file_info":{},"validations":[],"version":12}

Harvest item id: 879076