Hugging Face's logo

bigcode
/
santacoder

Text Generation
Transformers
PyTorch
code
gpt2
custom_code
Eval Results (legacy)
text-generation-inference
Model card Files
xet
 Community
45
harmdevries commited on
Commit
d3c3868
·
1 Parent(s): e02c7b7

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +28 -24
README.md CHANGED
@@ -20,38 +20,39 @@ model-index:
20
  - task:
21
  type: text-generation
22
  dataset:
23
- type: openai_humaneval
24
- name: HumanEval (Python)
25
  metrics:
26
  - name: pass@1
27
  type: pass@1
28
- value: 0.0
29
  verified: false
30
  - name: pass@10
31
  type: pass@10
32
- value: 0.0
33
  verified: false
34
  - name: pass@100
35
  type: pass@100
36
- value: 0.0
37
  verified: false
38
  - task:
39
  type: text-generation
40
  dataset:
41
  type: nuprl/MultiPL-E
42
- name: MultiPL HumanEval (Java)
43
  metrics:
44
  - name: pass@1
45
  type: pass@1
46
- value: 0.0
47
  verified: false
48
  - name: pass@10
49
  type: pass@10
50
- value: 0.0
51
  verified: false
52
  - name: pass@100
53
  type: pass@100
54
- value: 0.41
 
55
  - task:
56
  type: text-generation
57
  dataset:
@@ -60,67 +61,70 @@ model-index:
60
  metrics:
61
  - name: pass@1
62
  type: pass@1
63
- value: 0.0
64
  verified: false
65
  - name: pass@10
66
  type: pass@10
67
- value: 0.0
68
  verified: false
69
  - name: pass@100
70
  type: pass@100
71
  value: 0.47
 
72
  - task:
73
  type: text-generation
74
  dataset:
75
- type: openai_humaneval
76
- name: MBPP (Python)
77
  metrics:
78
  - name: pass@1
79
  type: pass@1
80
- value: 0.0
81
  verified: false
82
  - name: pass@10
83
  type: pass@10
84
- value: 0.0
85
  verified: false
86
  - name: pass@100
87
  type: pass@100
88
- value: 0.49
89
  verified: false
90
  - task:
91
  type: text-generation
92
  dataset:
93
  type: nuprl/MultiPL-E
94
- name: MultiPL MBPP (Java)
95
  metrics:
96
  - name: pass@1
97
  type: pass@1
98
- value: 0.0
99
  verified: false
100
  - name: pass@10
101
  type: pass@10
102
- value: 0.0
103
  verified: false
104
  - name: pass@100
105
  type: pass@100
106
- value: 0.0
 
107
  - task:
108
  type: text-generation
109
  dataset:
110
  type: nuprl/MultiPL-E
111
- name: MultiPL MBPP (JavaScript)
112
  metrics:
113
  - name: pass@1
114
  type: pass@1
115
- value: 0.0
116
  verified: false
117
  - name: pass@10
118
  type: pass@10
119
- value: 0.0
120
  verified: false
121
  - name: pass@100
122
  type: pass@100
123
- value: 0.0
 
124
  - task:
125
  type: text-generation
126
  dataset:
 
20
  - task:
21
  type: text-generation
22
  dataset:
23
+ type: nuprl/MultiPL-E
24
+ name: MultiPL HumanEval (Python)
25
  metrics:
26
  - name: pass@1
27
  type: pass@1
28
+ value: 0.18
29
  verified: false
30
  - name: pass@10
31
  type: pass@10
32
+ value: 0.29
33
  verified: false
34
  - name: pass@100
35
  type: pass@100
36
+ value: 0.49
37
  verified: false
38
  - task:
39
  type: text-generation
40
  dataset:
41
  type: nuprl/MultiPL-E
42
+ name: MultiPL MBPP (Python)
43
  metrics:
44
  - name: pass@1
45
  type: pass@1
46
+ value: 0.35
47
  verified: false
48
  - name: pass@10
49
  type: pass@10
50
+ value: 0.58
51
  verified: false
52
  - name: pass@100
53
  type: pass@100
54
+ value: 0.77
55
+ verified: false
56
  - task:
57
  type: text-generation
58
  dataset:
 
61
  metrics:
62
  - name: pass@1
63
  type: pass@1
64
+ value: 0.16
65
  verified: false
66
  - name: pass@10
67
  type: pass@10
68
+ value: 0.27
69
  verified: false
70
  - name: pass@100
71
  type: pass@100
72
  value: 0.47
73
+ verified: false
74
  - task:
75
  type: text-generation
76
  dataset:
77
+ type: nuprl/MultiPL-E
78
+ name: MultiPL MBPP (Javascript)
79
  metrics:
80
  - name: pass@1
81
  type: pass@1
82
+ value: 0.28
83
  verified: false
84
  - name: pass@10
85
  type: pass@10
86
+ value: 0.51
87
  verified: false
88
  - name: pass@100
89
  type: pass@100
90
+ value: 0.70
91
  verified: false
92
  - task:
93
  type: text-generation
94
  dataset:
95
  type: nuprl/MultiPL-E
96
+ name: MultiPL HumanEval (Java)
97
  metrics:
98
  - name: pass@1
99
  type: pass@1
100
+ value: 0.15
101
  verified: false
102
  - name: pass@10
103
  type: pass@10
104
+ value: 0.26
105
  verified: false
106
  - name: pass@100
107
  type: pass@100
108
+ value: 0.41
109
+ verified: false
110
  - task:
111
  type: text-generation
112
  dataset:
113
  type: nuprl/MultiPL-E
114
+ name: MultiPL MBPP (Java)
115
  metrics:
116
  - name: pass@1
117
  type: pass@1
118
+ value: 0.28
119
  verified: false
120
  - name: pass@10
121
  type: pass@10
122
+ value: 0.44
123
  verified: false
124
  - name: pass@100
125
  type: pass@100
126
+ value: 0.59
127
+ verified: false
128
  - task:
129
  type: text-generation
130
  dataset: