-
Notifications
You must be signed in to change notification settings - Fork 2
/
eval.py
102 lines (80 loc) · 3.72 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "7"
import argparse
from lm_eval import evaluator
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer
from awq.evaluation import (
evaluate_perplexity,
eval_librispeech,
eval_mmlu,
eval_humaneval,
eval_kl_divergence,
)
def run_eval(
model_path, quant_file, device, tasks, task_batch_size, task_n_shot,
task_use_pretrained, pretrained_safetensors
):
"""
Post quantization: Evaluate perplexity on wikitext with EleutherAI Evaluation Harness
"""
tasks = tasks.split(',')
# Load model
if len(tasks) == 1 and tasks[0] != "mmlu" and tasks[0] != "librispeech":
if task_use_pretrained:
model = AutoAWQForCausalLM.from_pretrained(model_path, safetensors=pretrained_safetensors)
else:
model = AutoAWQForCausalLM.from_quantized(model_path, quant_file, fuse_layers=False)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
# Load adapter
if len(tasks) == 1 and tasks[0] == 'wikitext':
evaluate_perplexity(model.model, tokenizer)
elif len(tasks) == 1 and tasks[0] == 'librispeech':
eval_librispeech(model_path)
elif len(tasks) == 1 and tasks[0] == 'mmlu':
eval_mmlu(model_path, task_n_shot, task_batch_size, device, task_use_pretrained)
elif len(tasks) == 1 and tasks[0] == 'humaneval':
eval_humaneval(model, tokenizer)
elif len(tasks) == 1 and tasks[0] == 'kldiv':
eval_kl_divergence(model.model, model.model, tokenizer, seqlen=1024)
else:
# Evaluate perplexity of quantized model
results = evaluator.simple_evaluate(
model=model,
tasks=tasks,
batch_size=task_batch_size,
no_cache=True,
num_fewshot=task_n_shot,
)
print(evaluator.make_table(results))
if __name__ == '__main__':
"""
- Run perplexity of quantized model:
python examples/eval.py --model_path casperhansen/mistral-7b-instruct-v0.1-awq
- Run perplexity unquantized FP16 model:
python examples/eval.py --use_pretrained --model_path lmsys/vicuna-7b-v1.5
- Run MMLU of quantized model:
python examples/eval.py --model_path TheBloke/zephyr-7B-beta-AWQ --tasks mmlu --n_shot 1 --batch_size 4
"""
parser = argparse.ArgumentParser()
parser.add_argument('--model_path', type=str, help='Path to hf model')
parser.add_argument('--quant_file', default='', type=str, help='Path to quantized AWQ model file')
parser.add_argument('--device', type=str, default='cuda:7', help='Device to load model to')
parser.add_argument("--use_pretrained", default=False, action='store_true',
help="Pass '--use_pretrained' to use a pretrained model running FP16")
parser.add_argument("--pretrained_safetensors", default=False, action='store_true',
help="Load safetensors for FP16 model")
parser.add_argument('--tasks', type=str, default='wikitext', help='Tasks to evaluate. '
'Separate tasks by comma for multiple tasks.'
'https://github.com/EleutherAI/lm-evaluation-harness/blob/master/docs/task_table.md')
parser.add_argument('--batch_size', type=int, default=1)
parser.add_argument('--n_shot', type=int, default=0)
args = parser.parse_args()
run_eval(
args.model_path, args.quant_file, args.device,
args.tasks, args.batch_size, args.n_shot, args.use_pretrained,
args.pretrained_safetensors
)
# python eval.py --model_path /data4/share/pingzhi/Llama-2-7b-chat-hf-awq-w_bit_2\
# git config user.name "Jinxiaolong1129"
# git config user.email "jinxiaolong1129@gmail.com"