Update scandeval_summary.py

scandeval_summary.py

@@ -1,17 +1,44 @@
+import argparse
 import pandas as pd
 
-df = pd.read_json("scandeval_benchmark_results.jsonl", lines=True)
+#Run ScandEval on model
+#scandeval --model-id KennethTM/bert-base-uncased-danish
 
-df["total"] = [i["total"] for i in df["results"]]
+#Format results as markdown
+#python scandeval_summary.py --name scandeval_benchmark_results.jsonl
 
-summary = []
-for i in df["total"]:
+def scandeval2md(file):
+    df = pd.read_json(file, lines=True)
 
-    summary_string = [f'{k.replace("test_", "")} = {round(v, 2)}' for k,v in i.items()]
+    df["total"] = [i["total"] for i in df["results"]]
 
-    summary.append(", ".join(summary_string))
+    summary = []
+    for i in df["total"]:
 
-df["summary"] = summary
+        summary_list = []
 
-md_table = df[["task", "dataset", "summary"]].to_markdown(index=False)
-print(md_table)
+        for k, v in i.items():
+            summary_string = ""
+
+            if "_se" in k:
+                summary_string += f" (±{round(v, 2)})\n"
+            else:
+                summary_string += f'{k.replace("test_", "")} = {round(v, 2)}'
+
+            summary_list.append(summary_string)
+        
+        summary.append("".join(summary_list).strip())
+
+    df["summary"] = summary
+
+    md_table = df[["task", "dataset", "summary"]].rename(columns={"task": "Task", "dataset": "Dataset", "summary": "Score (±SE)"}).to_markdown(index=False)
+
+    print(md_table)
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description='Script for formatting ScandEval .jsonl result file as markdown to standard output')
+    parser.add_argument('--name', help='ScandEval .jsonl result file')
+    args = parser.parse_args()
+
+    scandeval2md(args.name)