/codex_vs_hackerrank

Code Synthesis Evaluation of Codex using Python problem statements from HackerRank

Primary LanguagePython

Experiments with Codex

Code Synthesis Evaluation of Codex using Python problem statements from HackerRank

01_intoduction

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#1 ✅ ➖ ➖ ➖
#2 ✅ ✅ ❌ ❌
#3 ✅ ✅ ✅ ➖
#4 ✅ ✅ ❌ ➖
#5 ✅ ➖ ➖ ➖
#6 ✅ ✅ ➖ ➖
#7 ✅ ✅ ✅ ➖

02_basic_data_types

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#8 ✅ ✅ ✅ ➖
#9 ✅ ✅ ➖ ❌
#10 ✅ ✅ ➖ ➖
#11 ✅ ✅ ✅ ➖
#12 ✅ ✅ ✅ ➖
#13 ✅ ➖ ➖ ➖

03_strings

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#14 ✅ ➖ ➖ ➖
#15 ✅ ➖ ➖ ❌
#16 ✅ ✅ ✅ ❌
#17 ✅ ✅ ✅ ❌
#18 ✅ ✅ ✅ ➖
#19 ✅ ✅ ❌ ➖
#20 ✅ ➖ ➖ ➖
#21 ✅ ✅ ✅ ➖
#22 ✅ ✅ ❌ ➖
#23 ✅ ✅ ❌ ➖
#24 ✅ ➖ ➖ ➖
#25 ✅ ➖ ➖ ➖
#26 ✅ ✅ ✅ ➖
#27 ✅ ✅ ✅ ➖

04_sets

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#28 ✅ ✅ ❌ ➖
#29 ✅ ✅ ❌ ➖
#30 ✅ ✅ ➖ ➖
#31 ✅ ✅ ✅ ➖
#32 ✅ ✅ ✅ ➖
#33 ✅ ✅ ✅ ➖
#34 ✅ ✅ ✅ ➖
#35 ✅ ✅ ❌ ➖
#36 ✅ ✅ ❌ ➖
#37 ✅ ✅ ✅ ❌
#38 ✅ ✅ ✅ ❌
#39 ✅ ❌ ❌ ➖
#40 ✅ ✅ ❌ ➖

05_math

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#41 ✅ ✅ ➖ ➖
#42 ✅ ✅ ❌ ➖
#43 ✅ ✅ ❌ ➖
#44 ✅ ✅ ❌ ❌
#45 ✅ ✅ ✅ ❌
#46 ✅ ✅ ➖ ❌
#47 ✅ ✅ ❌ ➖

06_itertools

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#48 ✅ ✅ ❌ ➖
#49 ✅ ✅ ❌ ➖
#50 ✅ ✅ ❌ ➖
#51 ✅ ✅ ❌ ➖
#52 ✅ ➖ ❌ ➖
#53 ✅ ❌ ❌ ➖
#54 ✅ ❌ ❌ ➖

07_collections

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#55 ✅ ✅ ✅ ➖
#56 ✅ ❌ ❌ ➖
#57 ✅ ✅ ✅ ➖
#58 ✅ ✅ ❌ ➖
#59 ✅ ✅ ❌ ➖
#60 ✅ ✅ ➖ ➖
#61 ✅ ✅ ❌ ➖
#62 ✅ ❌ ❌ ➖

08_date_and_time

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#63 ✅ ✅ ➖ ➖
#64 ✅ ✅ ❌ ➖

09_errors_and_exceptions

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#65 ✅ ❌ ❌ ➖
#66 ✅ ✅ ❌ ➖

10_classes

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#67 ✅ ✅ ➖ ➖
#68 ✅ ✅ ❌ ➖

11_builtins

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#69 ✅ ✅ ✅ ➖
#70 ✅ ❌ ❌ ➖
#71 ✅ ➖ ➖ ✅
#72 ✅ ✅ ❌ ➖
#73 ✅ ✅ ✅ ❌
#74 ✅ ➖ ❌ ➖

12_python_functionals

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#75 ✅ ➖ ➖ ✅
#76 ✅ ✅ ❌ ➖
#77 ✅ ✅ ➖ ➖

13_regex_and_parsing

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#78 ✅ ❌ ❌ ➖
#79 ✅ ➖ ❌ ➖
#80 ✅ ❌ ❌ ➖
#81 ✅ ✅ ❌ ➖
#82 ✅ ✅ ❌ ➖
#83 ✅ ✅ ✅ ❌
#84 ✅ ✅ ➖ ➖
#85 ✅ ✅ ➖ ➖
#86 ✅ ❌ ❌ ➖
#87 ✅ ❌ ❌ ➖
#88 ✅ ✅ ✅ ➖
#89 ✅ ✅ ❌ ➖
#90 ✅ ✅ ❌ ➖
#91 ✅ ✅ ❌ ❌
#92 ✅ ✅ ✅ ➖
#93 ✅ ✅ ✅ ➖
#94 ✅ ❌ ❌ ➖

14_xml

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#95 ✅ ✅ ➖ ➖
#96 ✅ ✅ ➖ ➖

15_closures_and_decorators

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#97 ✅ ❌ ❌ ➖
#98 ✅ ✅ ✅ ➖

16_numpy

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#99 ✅ ➖ ❌ ✅
#100 ✅ ✅ ✅ ❌
#101 ✅ ✅ ✅ ➖
#102 ✅ ✅ ✅ ➖
#103 ✅ ➖ ➖ ➖
#104 ✅ ✅ ➖ ➖
#105 ✅ ✅ ✅ ➖
#106 ✅ ✅ ➖ ➖
#107 ✅ ✅ ✅ ❌
#108 ✅ ✅ ❌ ❌
#109 ✅ ❌ ❌ ➖
#110 ✅ ✅ ✅ ❌
#111 ✅ ✅ ❌ ➖
#112 ✅ ❌ ❌ ➖
#113 ✅ ❌ ❌ ➖

17_debugging

problem_statement full_problems missing_specifications missing_objectives diff_objectives
#114 ✅ ✅ ❌ ➖
#115 ✅ ✅ ❌ ➖