| { |
| "completion": [ |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-4o-2024-05-13", |
| "Pass_at_1": 1.0 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.9378571429 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.9357142857 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.9214285714 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.8642857143 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.8535714286 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.8471428571 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.845 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.8392857143 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.8192857143 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.7964285714 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.7485714286 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.6378571429 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.5992857143 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.5971428571 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.5057142857 |
| } |
| ], |
| "compilation_class_wise": [ |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.7942857143 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.7414285714 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.725 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.7171428571 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.7 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.6978571429 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.6828571429 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.6814285714 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.6764285714 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-4o-2024-05-13", |
| "Pass_at_1": 0.6607142857 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.58 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.5778571429 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.5021428571 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.4985714286 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.4335714286 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.3507142857 |
| } |
| ], |
| "compilation_test_wise": [ |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-4o-2024-05-13", |
| "Pass_at_1": 0.5035714286 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.4202826585 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.3443277311 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.3378676471 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.3183823529 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.3121848739 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.2836134454 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.2621848739 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.2240546218 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.1930147059 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.0955357143 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.0932773109 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.0701680672 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.0352941176 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.0 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.0 |
| } |
| ], |
| "pass_class_wise": [ |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.7832360347 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.723699056 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.7036481325 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.6855203826 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.6846699639 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.6808480861 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.6772858762 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.6547244155 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.6547232007 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-4o-2024-05-13", |
| "Pass_at_1": 0.6545897285 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.5674101922 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.5603969625 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.4878321662 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.4863639752 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.4261740357 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.3468474087 |
| } |
| ], |
| "pass_test_wise": [ |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-4o-2024-05-13", |
| "Pass_at_1": 0.3438179726 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.3047552867 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.2941156144 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.2544265255 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.2224382166 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.2083516025 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.2028454735 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-6.7b-instruct", |
| "Pass_at_1": 0.1967821219 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.1930147059 |
| }, |
| { |
| "Context": "selective", |
| "Method": "holistic", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.1669267449 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.0714792814 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "Phind-CodeLlama-34B-v2", |
| "Pass_at_1": 0.061012122 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "gpt-3.5-turbo-1106", |
| "Pass_at_1": 0.0514928193 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "deepseek-coder-33b-instruct", |
| "Pass_at_1": 0.0350620781 |
| }, |
| { |
| "Context": "selective", |
| "Method": "independent", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.0 |
| }, |
| { |
| "Context": "selective", |
| "Method": "incremental", |
| "Model": "WizardCoder-15B-V1.0", |
| "Pass_at_1": 0.0 |
| } |
| ] |
| } |
