import json
import pandas as pd
import numpy as np


with open(r"D:/code/score_test/data/nlt_info_20230926.log",
          "r",
          encoding='ANSI') as f:
    data = f.read()
    data = data.split('\n')
    count = 0
    C = 0
    all = 0
    json_list = []
    for i in range(len(data)):
        all += 1
        temp = '{' + data[i] + '}'
        d = temp.find('"recommend')
        e = temp.find('"recommendlist')
        if d < 0:
            continue
        if d > 0 and e > 0:
            continue
        t = json.loads(temp)
        json_list.append(t)
    data1 = pd.DataFrame(json_list)
data2 = pd.read_csv('D:/code/score_test/data/20230829评分系统材料.csv')

questionDict = {}
for i in range(data2.shape[0]):
    temp = data2.loc[i, 'question']
    temp_answer = data2.loc[i, 'expected']
    if not temp_answer == '空':
        expected_list = temp_answer.split('/')
    else:
        expected_list = []
    questionDict[temp] = expected_list


dataw = []
for i in range(data1.shape[0]):
    temp = data1.loc[i, 'question']
    temp_answer = data1.loc[i, 'answers']
    for j in range(len(temp_answer)):
        if temp_answer[j][0] in questionDict[temp]:
            dataw.append([temp, temp_answer[j][0], 1])
        else:
            dataw.append([temp, temp_answer[j][0], 0])

datas = []
for i in dataw:
    if i not in datas:
        datas.append(i)
print(len(datas))
dd = pd.DataFrame(datas, columns=['question', 'answer', 'label'])
dd.to_csv('data.csv', encoding="utf_8_sig")
