|
|
import csv |
|
|
import os |
|
|
|
|
|
import pytest |
|
|
|
|
|
output_path = 'regression_result' |
|
|
model = 'internlm2-chat-7b-hf' |
|
|
dataset = 'siqa' |
|
|
|
|
|
|
|
|
@pytest.fixture() |
|
|
def result_scores(): |
|
|
file = find_csv_files(output_path) |
|
|
if file is None: |
|
|
return None |
|
|
return read_csv_file(file) |
|
|
|
|
|
|
|
|
@pytest.mark.usefixtures('result_scores') |
|
|
class TestChatScore: |
|
|
"""Test cases for chat model.""" |
|
|
|
|
|
def test_model_dataset_score(self, result_scores): |
|
|
result_score = result_scores.get(model).get(dataset) |
|
|
assert_score(result_score, 79.53) |
|
|
|
|
|
|
|
|
def assert_score(score, baseline): |
|
|
if score is None or score == '-': |
|
|
assert False, 'value is none' |
|
|
if float(score) < (baseline * 1.03) and float(score) > (baseline * 0.97): |
|
|
print(score + ' between ' + str(baseline * 0.97) + ' and ' + |
|
|
str(baseline * 1.03)) |
|
|
assert True |
|
|
else: |
|
|
assert False, score + ' not between ' + str( |
|
|
baseline * 0.97) + ' and ' + str(baseline * 1.03) |
|
|
|
|
|
|
|
|
def find_csv_files(directory): |
|
|
csv_files = [] |
|
|
for root, dirs, files in os.walk(directory): |
|
|
for file in files: |
|
|
if file.endswith('.csv'): |
|
|
csv_files.append(os.path.join(root, file)) |
|
|
if len(csv_files) > 1: |
|
|
raise 'have more than 1 result file, please check the result manually' |
|
|
if len(csv_files) == 0: |
|
|
return None |
|
|
return csv_files[0] |
|
|
|
|
|
|
|
|
def read_csv_file(file_path): |
|
|
with open(file_path, 'r') as csvfile: |
|
|
reader = csv.DictReader(csvfile) |
|
|
filtered_data = [] |
|
|
|
|
|
for row in reader: |
|
|
filtered_row = { |
|
|
k: v |
|
|
for k, v in row.items() |
|
|
if k not in ['version', 'metric', 'mode'] |
|
|
} |
|
|
filtered_data.append(filtered_row) |
|
|
|
|
|
result = {} |
|
|
for data in filtered_data: |
|
|
dataset = data.get('dataset') |
|
|
for key in data.keys(): |
|
|
if key == 'dataset': |
|
|
continue |
|
|
else: |
|
|
if key in result.keys(): |
|
|
result.get(key)[dataset] = data.get(key) |
|
|
else: |
|
|
result[key] = {dataset: data.get(key)} |
|
|
return result |
|
|
|
