-
Notifications
You must be signed in to change notification settings - Fork 3
/
analyze.py
37 lines (30 loc) · 1.06 KB
/
analyze.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import os
import json
import numpy as np
from collections import defaultdict
from pprint import pprint
from process import retrieve_knowledges
def getusername():
dirs = ['data/paper_test', 'data/train', 'data/test_B']
names = set()
for dir in dirs:
with open(os.path.join(dir, 'data.txt'), encoding='utf-8') as f:
for line in f.readlines():
info = json.loads(line, encoding='utf-8')
names.add(info['user_profile']['姓名'])
return names
def getmaskword(attr):
dirs = ['data/paper_test', 'data/train']
words = set()
for dir in dirs:
with open(os.path.join(dir, 'data.txt'), encoding='utf-8') as f:
for line in f.readlines():
num = 0
info = json.loads(line, encoding='utf-8')
for kb in info['knowledge']:
if attr in kb[1]:
num += 1
words.add(kb[2])
if num > 1:
print("error")
return words