File size: 751 Bytes
d988382 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | #!/usr/bin/env python
import kenlm
from datasets import load_dataset
from tqdm import tqdm
def pp(log_score, length):
return 10.0 ** (-log_score / length)
# http://dl.fbaipublicfiles.com/cc_net/lm/es.arpa.bin
model = kenlm.Model("es.arpa.bin")
mc4 = load_dataset("mc4", "es", streaming=True)
with open("mc4-es-perplexity.txt", "w") as f:
for sample in tqdm(mc4["train"].shuffle(buffer_size=100_000), total=416057992):
lines = sample["text"].split("\n")
doc_log_score, doc_length = 0, 0
for line in lines:
log_score = model.score(line)
length = len(line.split()) + 1
doc_log_score += log_score
doc_length += length
f.write(f"{pp(doc_log_score, doc_length)}\n")
|