Skip to content

Commit 6997990

Browse files
tonysytonysy
authoredSep 12, 2024··
[Feature] Update Models (#1518)
* Update Models * Update * Update humanevalx * Update * Update
·
0.4.10.3.3
1 parent 3754dc1 commit 6997990

21 files changed

+225
-7
lines changed
 

‎configs/models/hf_llama/hf_llama3_1_70b_instruct.py

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='llama-3_1-70b-instruct-hf',
7+
path='meta-llama/Meta-Llama-3.1-70B-Instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=4),
11+
stop_words=['<|end_of_text|>', '<|eot_id|>'],
12+
)
13+
]

‎configs/models/hf_llama/hf_llama3_1_8b_instruct.py

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='llama-3_1-8b-instruct-hf',
7+
path='meta-llama/Meta-Llama-3.1-8B-Instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
stop_words=['<|end_of_text|>', '<|eot_id|>'],
12+
)
13+
]

‎configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from opencompass.models import TurboMindModelwithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=TurboMindModelwithChatTemplate,
6+
abbr='llama-3_1-70b-instruct-turbomind',
7+
path='meta-llama/Meta-Llama-3.1-70B-Instruct',
8+
engine_config=dict(max_batch_size=16, tp=4),
9+
gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),
10+
max_seq_len=7168,
11+
max_out_len=1024,
12+
batch_size=16,
13+
run_cfg=dict(num_gpus=4),
14+
stop_words=['<|end_of_text|>', '<|eot_id|>'],
15+
)
16+
]

‎configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
models = [
44
dict(
55
type=TurboMindModelwithChatTemplate,
6-
abbr='llama-3.1-8b-instruct-turbomind',
6+
abbr='llama-3_1-8b-instruct-turbomind',
77
path='meta-llama/Meta-Llama-3.1-8B-Instruct',
88
engine_config=dict(max_batch_size=16, tp=1),
99
gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),

‎configs/models/mistral/vllm_mixtral_large_instruct_2407.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from opencompass.models import VLLMwithChatTemplate
2+
3+
4+
models = [
5+
dict(
6+
type=VLLMwithChatTemplate,
7+
abbr='mixtral-large-instruct-2407-vllm',
8+
path='mistralai/Mistral-Large-Instruct-2407',
9+
model_kwargs=dict(tensor_parallel_size=8),
10+
max_out_len=256,
11+
batch_size=16,
12+
generation_kwargs=dict(temperature=0),
13+
run_cfg=dict(num_gpus=8),
14+
)
15+
]

‎configs/models/openbmb/hf_minicpm3_4b.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='MiniCPM3-4B-hf',
7+
path='openbmb/MiniCPM3-4B',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
model_kwargs=dict(
12+
torch_dtype='torch.bfloat16',
13+
),
14+
)
15+
]

‎configs/models/phi/hf_phi_3_5_MoE_instruct.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='phi-3-5-MoE-instruct-hf',
7+
path='microsoft/Phi-3.5-MoE-instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
)
12+
]

‎configs/models/phi/hf_phi_3_5_mini_instruct.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='phi-3-5-mini-instruct-hf',
7+
path='microsoft/Phi-3.5-mini-instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
)
12+
]

‎configs/summarizers/groups/humanevalx.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
humanevalx_summary_groups = []
2+
3+
_humanevalx_all = ['python', 'cpp', 'go', 'java', 'js']
4+
_humanevalx_all = ['humanevalx-' + d for d in _humanevalx_all]
5+
humanevalx_summary_groups.append({'name': 'humanevalx', 'subsets': _humanevalx_all})

‎opencompass/configs/models/hf_llama/hf_llama3_1_70b_instruct.py

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='llama-3_1-70b-instruct-hf',
7+
path='meta-llama/Meta-Llama-3.1-70B-Instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=4),
11+
stop_words=['<|end_of_text|>', '<|eot_id|>'],
12+
)
13+
]

‎opencompass/configs/models/hf_llama/hf_llama3_1_8b_instruct.py

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='llama-3_1-8b-instruct-hf',
7+
path='meta-llama/Meta-Llama-3.1-8B-Instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
stop_words=['<|end_of_text|>', '<|eot_id|>'],
12+
)
13+
]

‎opencompass/configs/models/hf_llama/lmdeploy_llama3_1_70b_instruct.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from opencompass.models import TurboMindModelwithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=TurboMindModelwithChatTemplate,
6+
abbr='llama-3_1-70b-instruct-turbomind',
7+
path='meta-llama/Meta-Llama-3.1-70B-Instruct',
8+
engine_config=dict(max_batch_size=16, tp=4),
9+
gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),
10+
max_seq_len=7168,
11+
max_out_len=1024,
12+
batch_size=16,
13+
run_cfg=dict(num_gpus=4),
14+
stop_words=['<|end_of_text|>', '<|eot_id|>'],
15+
)
16+
]

‎opencompass/configs/models/hf_llama/lmdeploy_llama3_1_8b_instruct.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
models = [
44
dict(
55
type=TurboMindModelwithChatTemplate,
6-
abbr='llama-3.1-8b-instruct-turbomind',
6+
abbr='llama-3_1-8b-instruct-turbomind',
77
path='meta-llama/Meta-Llama-3.1-8B-Instruct',
88
engine_config=dict(max_batch_size=16, tp=1),
99
gen_config=dict(top_k=1, temperature=1e-6, top_p=0.9, max_new_tokens=1024),

‎opencompass/configs/models/mistral/vllm_mixtral_large_instruct_2407.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from opencompass.models import VLLMwithChatTemplate
2+
3+
4+
models = [
5+
dict(
6+
type=VLLMwithChatTemplate,
7+
abbr='mixtral-large-instruct-2407-vllm',
8+
path='mistralai/Mistral-Large-Instruct-2407',
9+
model_kwargs=dict(tensor_parallel_size=8),
10+
max_out_len=256,
11+
batch_size=16,
12+
generation_kwargs=dict(temperature=0),
13+
run_cfg=dict(num_gpus=8),
14+
)
15+
]

‎opencompass/configs/models/openbmb/hf_minicpm3_4b.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='MiniCPM3-4B-hf',
7+
path='openbmb/MiniCPM3-4B',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
model_kwargs=dict(
12+
torch_dtype='torch.bfloat16',
13+
),
14+
)
15+
]

‎opencompass/configs/models/phi/hf_phi_3_5_MoE_instruct.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='phi-3-5-MoE-instruct-hf',
7+
path='microsoft/Phi-3.5-MoE-instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
)
12+
]

‎opencompass/configs/models/phi/hf_phi_3_5_mini_instruct.py

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from opencompass.models import HuggingFacewithChatTemplate
2+
3+
models = [
4+
dict(
5+
type=HuggingFacewithChatTemplate,
6+
abbr='phi-3-5-mini-instruct-hf',
7+
path='microsoft/Phi-3.5-mini-instruct',
8+
max_out_len=1024,
9+
batch_size=8,
10+
run_cfg=dict(num_gpus=1),
11+
)
12+
]

‎opencompass/configs/summarizers/groups/humanevalx.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
humanevalx_summary_groups = []
2+
3+
_humanevalx_all = ['python', 'cpp', 'go', 'java', 'js']
4+
_humanevalx_all = ['humanevalx-' + d for d in _humanevalx_all]
5+
humanevalx_summary_groups.append({'name': 'humanevalx', 'subsets': _humanevalx_all})

‎opencompass/datasets/ds1000.py

+7-2
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ class DS1000ServiceEvaluator(BaseEvaluator):
366366
def __init__(self,
367367
lib: str,
368368
ip_address='localhost',
369-
port=5000,
369+
port='',
370370
timeout=600) -> None:
371371
assert lib in _LIBRARY_NAME_LIST, (
372372
f' lib must be in {_LIBRARY_NAME_LIST}')
@@ -421,9 +421,14 @@ def _code_eval_service(self, file_path: str) -> tuple:
421421
Returns:
422422
tuple[bool, str]: Whether the access is successful and the output.
423423
"""
424+
if self.port:
425+
eval_server_url = f'{self.ip_address}:{self.port}/evaluate'
426+
else:
427+
eval_server_url = f'{self.ip_address}/evaluate'
428+
424429
exec_result = subprocess.run([
425430
'curl', '-X', 'POST', '-F', f'file=@{file_path}',
426-
f'{self.ip_address}:{self.port}/evaluate'
431+
f'{eval_server_url}'
427432
],
428433
timeout=self.timeout,
429434
capture_output=True)

‎opencompass/datasets/humanevalx.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ class HumanevalXEvaluator(BaseEvaluator):
7676
def __init__(self,
7777
language,
7878
ip_address='localhost',
79-
port=5000,
79+
port='',
8080
retry=2,
8181
timeout=600) -> None:
8282
assert language in _LANGUAGE_NAME_DICT.keys(), (
@@ -141,10 +141,13 @@ def score(self, predictions, references):
141141
f'\nError Information: {output}')
142142

143143
def _code_eval_service(self, file_path):
144+
if self.port:
145+
eval_server_url = f'{self.ip_address}:{self.port}/evaluate'
146+
else:
147+
eval_server_url = f'{self.ip_address}/evaluate'
144148
exec_result = subprocess.run([
145149
'curl', '-X', 'POST', '-F', f'file=@{file_path}', '-F',
146-
f'dataset=humanevalx/{self.language}',
147-
f'{self.ip_address}:{self.port}/evaluate'
150+
f'dataset=humanevalx/{self.language}', f'{eval_server_url}'
148151
],
149152
timeout=self.timeout,
150153
capture_output=True)

‎opencompass/utils/datasets_info.py

+8
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,14 @@
340340
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/humaneval.zip",
341341
"md5": "88b1b89dc47b7121c81da6bcd85a69c3",
342342
},
343+
"/humanevalx": {
344+
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/humanevalx.zip",
345+
"md5": "22930355c03fb73fb5bae14b50f1deb9",
346+
},
347+
"/ds1000_data": {
348+
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/ds1000_data.zip",
349+
"md5": "1a4990aec04a2fd73ccfad12e2d43b43",
350+
},
343351
"/drop_simple_eval/": {
344352
"url": "http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/drop_simple_eval.zip",
345353
"md5": "c912afe5b4a63509851cf16e6b91830e",

0 commit comments

Comments
 (0)
Please sign in to comment.