Fixed cutom model in legel doc

datasets/legal_sample.jsonl

@@ -0,0 +1,3 @@
+{"question":"Summarize the confidentiality clause: The parties agree to keep all proprietary information confidential for five years.","answer":"Both parties must keep proprietary info secret for five years."}
+{"question":"Summarize the termination clause: Either party may terminate with 30 days written notice without cause.","answer":"Either side can end the agreement with 30 days written notice."}
+{"question":"Summarize the liability clause: Liability is limited to direct damages not exceeding fees paid in the last 12 months.","answer":"Each party's liability is capped to direct damages up to fees from the past year."}

legaldoc_summarizer/app.py

@@ -1,3 +1,4 @@
+import os
 import streamlit as st
 from shared.hf_helpers import build_pipeline
 import yaml
@@ -10,13 +11,29 @@ def main():
     with open(CONFIG_PATH) as f:
         cfg = yaml.safe_load(f)
 
-    model_name = st.selectbox("Model:", [cfg["base_model"], "models/legaldoc_summarizer"])
+    base_model = cfg["base_model"]
+    finetuned_model = cfg.get("finetuned_model") or os.getenv("LEGALDOC_MODEL_ID")
+    local_model_dir = Path(cfg.get("finetuned_local_dir", "models/legaldoc_summarizer"))
+
+    model_options = [base_model]
+    if finetuned_model:
+        model_options.append(finetuned_model)
+    elif local_model_dir.exists():
+        model_options.append(str(local_model_dir))
+    else:
+        st.info(
+            "Using the base model until a fine-tuned checkpoint is available. "
+            "Train a model to populate `models/legaldoc_summarizer` or set `LEGALDOC_MODEL_ID` / `finetuned_model`."
+        )
+
+    model_name = st.selectbox("Model:", model_options)
+    hf_token = os.getenv("HUGGINGFACEHUB_API_TOKEN") or os.getenv("HF_TOKEN")
 
     @st.cache_resource
-    def get_pipeline(model_name):
-        return build_pipeline(model_name)
+    def get_pipeline(model_name, token):
+        return build_pipeline(model_name, token=token)
 
-    pipe = get_pipeline(model_name)
+    pipe = get_pipeline(model_name, hf_token)
 
     st.write("Paste a contract clause or judgment text below:")
     text = st.text_area("Clause or Legal Text", height=250)

legaldoc_summarizer/config.yaml

@@ -1,5 +1,7 @@
 project: "LegalDocSummarizer"
 base_model: "google/flan-t5-base"
+finetuned_model: ""        # Optional: HF repo ID for a private/public fine-tuned model
+finetuned_local_dir: "models/legaldoc_summarizer"
 dataset_name: "cuad"   # Contract Understanding Atticus Dataset
 train:
   epochs: 3

legaldoc_summarizer/dataset_loader.py

@@ -1,18 +1,38 @@
+import os
+import pandas as pd
 from datasets import load_dataset
-import pandas as pd, os
+
 
 def load_legal_dataset():
     """
     Loads a small portion of the CUAD dataset (contract clauses).
-    Converts each clause into (document_text, summary) pairs.
+    Falls back to a tiny synthetic sample if the dataset is unavailable (e.g., offline).
     """
-    dataset = load_dataset("cuad", "cuad_v1", split="train[:200]")
-    df = pd.DataFrame(dataset)
-
-    df["question_text"] = "Summarize the key legal clause: " + df["question_text"]
-    df["answer"] = df["answers"].apply(lambda a: a[0]["text"][0] if a and a[0]["text"] else "")
+    try:
+        dataset = load_dataset("cuad", "cuad_v1", split="train[:200]")
+        df = pd.DataFrame(dataset)
+        df["question_text"] = "Summarize the key legal clause: " + df["question_text"]
+        df["answer"] = df["answers"].apply(lambda a: a[0]["text"][0] if a and a[0]["text"] else "")
+        data = df[["question_text", "answer"]].rename(columns={"question_text": "question"})
+    except Exception as exc:  # pragma: no cover - offline/sandbox fallback
+        print(f"⚠️ Unable to load CUAD from Hub ({exc}). Using synthetic sample.")
+        data = pd.DataFrame(
+            [
+                {
+                    "question": "Summarize the confidentiality clause: The parties agree to keep all proprietary information confidential for five years.",
+                    "answer": "Both parties must keep proprietary info secret for five years.",
+                },
+                {
+                    "question": "Summarize the termination clause: Either party may terminate with 30 days written notice without cause.",
+                    "answer": "Either side can end the agreement with 30 days written notice.",
+                },
+                {
+                    "question": "Summarize the liability clause: Liability is limited to direct damages not exceeding fees paid in the last 12 months.",
+                    "answer": "Each party's liability is capped to direct damages up to fees from the past year.",
+                },
+            ]
+        )
 
-    data = df[["question_text", "answer"]].rename(columns={"question_text": "question"})
     os.makedirs("datasets", exist_ok=True)
     data.to_json("datasets/legal_sample.jsonl", orient="records", lines=True)
     print("✅ Saved sample dataset to datasets/legal_sample.jsonl")

legaldoc_summarizer/evaluate.py

@@ -1,16 +1,63 @@
 import json
+import os
+import sys
+from pathlib import Path
 from datasets import load_dataset
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
-from shared.metrics import compute_rouge, compute_bleu, factuality_score
-from shared.utils import print_banner
 
-def evaluate_model(model_path="models/legaldoc_summarizer"):
+# Ensure repo root is on path when running directly
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from shared.metrics import compute_rouge, compute_bleu, factuality_score  # noqa: E402
+from shared.utils import print_banner, load_yaml_config  # noqa: E402
+from legaldoc_summarizer.dataset_loader import load_legal_dataset  # noqa: E402
+
+
+def _resolve_model_id(cfg):
+    finetuned = cfg.get("finetuned_model") or os.getenv("LEGALDOC_MODEL_ID")
+    local_dir = Path(cfg.get("finetuned_local_dir", "models/legaldoc_summarizer"))
+    if finetuned:
+        return finetuned
+    if local_dir.exists():
+        return str(local_dir)
+    return cfg["base_model"]
+
+
+def _build_hf_kwargs(token: str | None) -> dict:
+    if not token:
+        return {}
+    return {"token": token}
+
+
+def _fallback_hf_kwargs(token: str | None) -> dict:
+    if not token:
+        return {}
+    return {"use_auth_token": token}
+
+
+def evaluate_model():
     print_banner("Evaluating LegalDoc Summarizer")
 
-    tokenizer = AutoTokenizer.from_pretrained(model_path)
-    model = AutoModelForSeq2SeqLM.from_pretrained(model_path)
+    cfg = load_yaml_config(Path(__file__).resolve().parent / "config.yaml")
+    model_id = _resolve_model_id(cfg)
+    auth_token = os.getenv("HUGGINGFACEHUB_API_TOKEN") or os.getenv("HF_TOKEN")
 
-    dataset = load_dataset("json", data_files="datasets/legal_sample.jsonl", split="train[:100]")
+    kwargs = _build_hf_kwargs(auth_token)
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(model_id, **kwargs)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **kwargs)
+    except TypeError:
+        fallback_kwargs = _fallback_hf_kwargs(auth_token)
+        tokenizer = AutoTokenizer.from_pretrained(model_id, **fallback_kwargs)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_id, **fallback_kwargs)
+
+    dataset_path = ROOT / "datasets/legal_sample.jsonl"
+    if not dataset_path.exists():
+        print_banner("Dataset not found. Creating sample dataset...")
+        load_legal_dataset()
+    dataset = load_dataset("json", data_files=str(dataset_path), split="train[:100]")
 
     preds, refs = [], []
     for row in dataset:
@@ -24,9 +71,12 @@ def evaluate_model(model_path="models/legaldoc_summarizer"):
     results.update(compute_bleu(preds, refs))
     results.update(factuality_score(preds, refs))
 
-    with open("models/legaldoc_summarizer/eval_results.json", "w") as f:
+    results_dir = ROOT / "models/legaldoc_summarizer"
+    results_dir.mkdir(parents=True, exist_ok=True)
+    with open(results_dir / "eval_results.json", "w") as f:
         json.dump(results, f, indent=2)
     print("✅ Evaluation complete:", results)
 
+
 if __name__ == "__main__":
     evaluate_model()

models/legaldoc_summarizer/eval_results.json

@@ -0,0 +1,82 @@
+{
+  "rouge1": [
+    [
+      0.14285714285714285,
+      0.0909090909090909,
+      0.1111111111111111
+    ],
+    [
+      0.30952380952380953,
+      0.31363636363636366,
+      0.3077441077441077
+    ],
+    [
+      0.5,
+      0.6,
+      0.5454545454545454
+    ]
+  ],
+  "rouge2": [
+    [
+      0.0,
+      0.0,
+      0.0
+    ],
+    [
+      0.06060606060606061,
+      0.07407407407407407,
+      0.06666666666666667
+    ],
+    [
+      0.1818181818181818,
+      0.2222222222222222,
+      0.19999999999999998
+    ]
+  ],
+  "rougeL": [
+    [
+      0.14285714285714285,
+      0.0909090909090909,
+      0.1111111111111111
+    ],
+    [
+      0.2857142857142857,
+      0.2928030303030303,
+      0.2855218855218855
+    ],
+    [
+      0.5,
+      0.6,
+      0.5454545454545454
+    ]
+  ],
+  "rougeLsum": [
+    [
+      0.14285714285714285,
+      0.0909090909090909,
+      0.1111111111111111
+    ],
+    [
+      0.2857142857142857,
+      0.2928030303030303,
+      0.2855218855218855
+    ],
+    [
+      0.5,
+      0.6,
+      0.5454545454545454
+    ]
+  ],
+  "bleu": 0.0,
+  "precisions": [
+    0.3333333333333333,
+    0.06666666666666667,
+    0.037037037037037035,
+    0.0
+  ],
+  "brevity_penalty": 0.9131007162822622,
+  "length_ratio": 0.9166666666666666,
+  "translation_length": 33,
+  "reference_length": 36,
+  "factuality": 0.3255411255411255
+}

requirements.txt

@@ -8,3 +8,9 @@ gradio==4.44.0
 tokenizers==0.19.1
 # Explicitly pin pydantic major to avoid breaking gradio deps
 pydantic==2.7.4
+# Evaluation and datasets
+datasets==2.21.0
+evaluate==0.4.2
+rouge-score==0.1.2
+nltk==3.9.1
+absl-py==2.1.0

retailgpt_evaluator/app.py

@@ -19,10 +19,30 @@ def main():
         cfg = yaml.safe_load(f)
 
     # Show leaderboard if exists
-    if os.path.exists("models/retail_eval_results.json"):
-        df = build_leaderboard()
-        st.subheader("📊 Model Leaderboard")
-        st.dataframe(df, use_container_width=True)
+    leaderboard_df = None
+    results_path = Path("models/retail_eval_results.json")
+    if results_path.exists():
+        try:
+            leaderboard_df = build_leaderboard(results_path)
+            st.subheader("📊 Model Leaderboard")
+            st.dataframe(leaderboard_df, use_container_width=True)
+
+            st.markdown("#### 📈 Evaluation Metrics")
+            metric_options = leaderboard_df["model"].tolist()
+            selected = st.selectbox("Inspect metrics for:", metric_options)
+            selected_row = leaderboard_df[leaderboard_df["model"] == selected].iloc[0]
+
+            cols = st.columns(4)
+            cols[0].metric("ROUGE-L", f"{selected_row['rougeL']:.3f}")
+            cols[1].metric("BLEU", f"{selected_row['bleu']:.3f}")
+            cols[2].metric("Factuality", f"{selected_row['factuality']:.3f}")
+            cols[3].metric("Score (avg)", f"{selected_row['score']:.3f}")
+
+            st.bar_chart(
+                leaderboard_df.set_index("model")[["rougeL", "bleu", "factuality", "score"]]
+            )
+        except Exception as exc:  # pragma: no cover - defensive UI fallback
+            st.error(f"Unable to load evaluation results: {exc}")
     else:
         st.warning("Run `evaluate.py` first to generate metrics.")
 

retailgpt_evaluator/evaluate.py

@@ -1,10 +1,19 @@
 import json
+import sys
+from pathlib import Path
 from datasets import load_dataset
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
-from shared.metrics import compute_rouge, compute_bleu, factuality_score
-from shared.utils import print_banner
 import torch
 
+# Ensure repo root is on the path so `shared` package is found when run directly
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from shared.metrics import compute_rouge, compute_bleu, factuality_score  # noqa: E402
+from shared.utils import print_banner, load_yaml_config  # noqa: E402
+from retailgpt_evaluator.dataset_loader import load_retail_dataset  # noqa: E402
+
 def run_eval_for_model(model_name, dataset):
     print_banner(f"Evaluating {model_name}")
     tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -22,9 +31,14 @@ def run_eval_for_model(model_name, dataset):
     return {"model": model_name, **r, **b, **f}
 
 def evaluate_all():
-    from shared.utils import load_yaml_config
-    cfg = load_yaml_config("config.yaml")
-    dataset = load_dataset("json", data_files="datasets/retail_sample.jsonl", split="train[:50]")
+    config_path = Path(__file__).resolve().parent / "config.yaml"
+    cfg = load_yaml_config(config_path)
+    dataset_path = ROOT / "datasets/retail_sample.jsonl"
+    if not dataset_path.exists():
+        print_banner("Dataset not found. Creating sample dataset...")
+        load_retail_dataset()
+    dataset = load_dataset("json", data_files=str(dataset_path), split="train[:50]")
+    (ROOT / "models").mkdir(exist_ok=True)
     results = [run_eval_for_model(m, dataset) for m in cfg["models"]]
     json.dump(results, open("models/retail_eval_results.json", "w"), indent=2)
     print("✅ Saved results to models/retail_eval_results.json")

shared/__init__.py

@@ -0,0 +1 @@
+# Makes shared a package so imports like `from shared import ...` work.

shared/hf_helpers.py

@@ -1,10 +1,42 @@
+import os
+from pathlib import Path
+from typing import Optional
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
 import torch
 
-def load_model_and_tokenizer(model_name: str):
+def _resolve_model_identifier(model_name: str) -> str:
+    """Return a valid model identifier or local path."""
+    path_candidate = Path(model_name)
+    if path_candidate.exists():
+        return str(path_candidate)
+    return model_name
+
+def _build_hub_kwargs(token: Optional[str]) -> dict:
+    """Prepare kwargs for Hugging Face Hub auth across library versions."""
+    if not token:
+        return {}
+    return {"token": token}
+
+def _fallback_hub_kwargs(token: Optional[str]) -> dict:
+    """Older transformers versions still expect use_auth_token."""
+    if not token:
+        return {}
+    return {"use_auth_token": token}
+
+def load_model_and_tokenizer(model_name: str, token: Optional[str] = None):
     """Load a model and tokenizer for inference."""
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+    resolved_model = _resolve_model_identifier(model_name)
+    auth_token = token or os.getenv("HUGGINGFACEHUB_API_TOKEN") or os.getenv("HF_TOKEN")
+
+    kwargs = _build_hub_kwargs(auth_token)
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(resolved_model, **kwargs)
+        model = AutoModelForSeq2SeqLM.from_pretrained(resolved_model, **kwargs)
+    except TypeError:
+        fallback_kwargs = _fallback_hub_kwargs(auth_token)
+        tokenizer = AutoTokenizer.from_pretrained(resolved_model, **fallback_kwargs)
+        model = AutoModelForSeq2SeqLM.from_pretrained(resolved_model, **fallback_kwargs)
+
     return model, tokenizer
 
 def generate_answer(model, tokenizer, prompt: str, max_tokens: int = 256):
@@ -14,6 +46,14 @@ def generate_answer(model, tokenizer, prompt: str, max_tokens: int = 256):
         outputs = model.generate(**inputs, max_new_tokens=max_tokens)
     return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
-def build_pipeline(model_name: str, task="text2text-generation"):
+def build_pipeline(model_name: str, task="text2text-generation", token: Optional[str] = None):
     """Return a Hugging Face pipeline for inference."""
-    return pipeline(task, model=model_name)
+    resolved_model = _resolve_model_identifier(model_name)
+    auth_token = token or os.getenv("HUGGINGFACEHUB_API_TOKEN") or os.getenv("HF_TOKEN")
+
+    kwargs = _build_hub_kwargs(auth_token)
+    try:
+        return pipeline(task, model=resolved_model, **kwargs)
+    except TypeError:
+        fallback_kwargs = _fallback_hub_kwargs(auth_token)
+        return pipeline(task, model=resolved_model, **fallback_kwargs)

shared/metrics.py

@@ -6,9 +6,11 @@ def compute_rouge(preds, refs):
     return rouge.compute(predictions=preds, references=refs)
 
 def compute_bleu(preds, refs):
+    """BLEU with simple whitespace tokenization for compatibility."""
     bleu = load_metric("bleu")
-    refs = [[r] for r in refs]  # bleu expects list of lists
-    return bleu.compute(predictions=preds, references=refs)
+    pred_tokens = [p.split() for p in preds]
+    ref_tokens = [[r.split()] for r in refs]  # bleu expects list of list of token lists
+    return bleu.compute(predictions=pred_tokens, references=ref_tokens)
 
 def factuality_score(preds, refs):
     """Very simple lexical overlap metric for factual alignment."""