Upload folder using huggingface_hub

config.json

@@ -28,5 +28,14 @@
   "label2id": {
     "INCOMPLETE": 0,
     "COMPLETE": 1
+  },
+  "custom_pipelines": {
+    "text-classification": {
+      "impl": "pipeline.ThoughtCompletionPipeline",
+      "pt": [
+        "DistilBertForSequenceClassification"
+      ],
+      "tf": []
+    }
   }
 }

pipeline.py

@@ -0,0 +1,110 @@
+
+import re
+import torch
+from transformers import Pipeline
+
+class ThoughtCompletionPipeline(Pipeline):
+    """Custom pipeline with linguistic rules for thought completion classification"""
+    
+    INCOMPLETE_ENDING_WORDS = [
+        'a', 'and', 'with', 'do', 'but', 'to', 'for', 'of', 'or', 'what', 'get',
+        'the', 'have', 'need', 'want', 'can', 'will', 'would', 'could', 'should',
+        'also', 'add', 'plus', 'oh', 'um', 'uh'
+    ]
+    
+    def _sanitize_parameters(self, **kwargs):
+        return {}, {}, {}
+    
+    def ends_with_incomplete_word(self, text):
+        """Check if text ends with words that indicate incomplete thought"""
+        text = text.strip().lower()
+        text = re.sub(r'[.,!?;]$', '', text)
+        words = text.split()
+        
+        if not words:
+            return False
+        
+        last_word = words[-1]
+        return last_word in self.INCOMPLETE_ENDING_WORDS
+    
+    def apply_linguistic_rules(self, text):
+        """Apply linguistic rules to determine if thought is incomplete"""
+        # Expecting format: "AI_utterance [SEP] CX_utterance"
+        parts = text.split('[SEP]')
+        if len(parts) != 2:
+            return None, None
+        
+        ai_utterance = parts[0].strip()
+        cx_utterance = parts[1].strip()
+        
+        # Check if customer utterance ends with incomplete word
+        if self.ends_with_incomplete_word(cx_utterance):
+            return [{'label': 'INCOMPLETE', 'score': 0.95}], 'rule:incomplete_ending'
+        
+        cx_lower = cx_utterance.lower()
+        
+        # Complete thought indicators
+        complete_phrases = [
+            "that's all", "nothing else", "i'm done", "that's it", 
+            "no thanks", "no thank you", "yes", "yes please",
+            "perfect", "great", "sounds good", "that's everything"
+        ]
+        
+        for phrase in complete_phrases:
+            if phrase in cx_lower:
+                return [{'label': 'COMPLETE', 'score': 0.95}], f'rule:contains_{phrase}'
+        
+        # Incomplete thought indicators
+        incomplete_phrases = [
+            "let me add", "actually", "wait", "i also need",
+            "can i get", "i want", "i'd like", "i need",
+            "hold on", "one more", "oh and"
+        ]
+        
+        for phrase in incomplete_phrases:
+            if phrase in cx_lower:
+                return [{'label': 'INCOMPLETE', 'score': 0.95}], f'rule:contains_{phrase}'
+        
+        return None, None
+    
+    def preprocess(self, inputs):
+        # First check rules
+        rule_result, rule_applied = self.apply_linguistic_rules(inputs)
+        if rule_result:
+            return {"rule_result": rule_result, "use_model": False}
+        
+        # If no rule applies, prepare for model
+        return {"inputs": inputs, "use_model": True}
+    
+    def _forward(self, model_inputs):
+        if not model_inputs["use_model"]:
+            return {"predictions": model_inputs["rule_result"]}
+        
+        # Use the model
+        inputs = self.tokenizer(
+            model_inputs["inputs"],
+            return_tensors="pt",
+            truncation=True,
+            padding=True,
+            max_length=128
+        )
+        
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+            predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
+            
+        return {"predictions": predictions}
+    
+    def postprocess(self, model_outputs):
+        if isinstance(model_outputs["predictions"], list):
+            # Rule-based result
+            return model_outputs["predictions"]
+        
+        # Model-based result
+        predictions = model_outputs["predictions"]
+        scores = predictions.numpy()[0]
+        
+        return [
+            {"label": "INCOMPLETE", "score": float(scores[0])},
+            {"label": "COMPLETE", "score": float(scores[1])}
+        ]