jstarks's picture
Upgrade to WSP V5 Two-Phase (13 GGUF LoRAs + r16 safetensors + Q6_K peak 178/270 & 5/20 tbench20 + Q4_K_M 176/270 & 86/150 STEM + xtools-at/mradermacher credits)
41de7a0 verified
Raw History Blame Contribute Delete
34.1 kB
"""13-Subspace Dynamic Weight-Space Router (V5 Two-Phase Reasoning-Boundary + Stateful Terminal)
tailored for xtools-at/Qwen3.5-9B-DeepThought-STEM.
Routes dynamically across all 13 rank-16 Woodbury GGUF LoRAs (0..12):
0: s1_think - Fast bounded </think> closure & verification compression
1: s2_restraint - Tool restraint (BFCL irrelevance + worker tools) & anti-sycophancy
2: s3_router - Multi-tool boundary routing
3: s4_mid_code - Mid-network algorithmic Python/code synthesis
4: s5_json_schema - Turn-2 strict JSON synthesis
5: s6_handoff - Unrecoverable escalation handoff gate (escalate_handoff)
6: s7_injection_guard - Indirect prompt-injection immunity
7: s8_grounding - Grounded context fidelity & missing-field abstention
8: s9_error_recovery - Recoverable local tool-error self-correction & loop breaking
9: s10_term_action - Stateful Terminal-Bench Turn-2 action/mutation continuity
10: s11_term_code - Mid-network compound bash & mutation synthesis
11: s12_stem_settle - Deep STEM/Competition-Math anti-loop </think> settling
12: s13_post_concise - Post-</think> concise termination (\boxed{ / Answer: / <|im_end|>)
Two-Phase Reasoning-Boundary WSP (phase_split=True):
- Phase 1 (<think>...</think>): Runs ONLY <think>-control subspaces (S1/S12),
never forcing </think> early.
- Phase 2 (after the model emits </think>): Switches /lora-adapters (~2 ms) to
S1=0.0, S12=0.0 and activates ONLY the task-specific Phase-2 subspace(s),
eliminating post-</think> </think> re-injection loops cleanly within budget.
"""
from __future__ import annotations
import json
import re
import sys
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
_HERE = Path(__file__).resolve().parent
PIPELINE_DIR = Path(r"C:\Users\jhjwi\.gemini\antigravity\scratch\wsp-5model-pipeline")
if (_HERE / "dynamic_router_9s.py").exists():
if str(_HERE) not in sys.path:
sys.path.insert(0, str(_HERE))
elif str(PIPELINE_DIR) not in sys.path:
sys.path.insert(0, str(PIPELINE_DIR))
from dynamic_router_9s import DynamicSubspaceRouter9S, RouteDecision, SUBSPACE_NAMES, _extract_text # noqa: E402
DT_SUBSPACE_NAMES = {
**SUBSPACE_NAMES,
11: "s12_stem_settle",
12: "s13_post_concise",
}
@dataclass
class DTRouteDecision(RouteDecision):
phase1_scales: Optional[List[float]] = None
phase2_scales: Optional[List[float]] = None
def _msgs_for_jinja(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Convert OpenAI JSON-string tool_calls.arguments to dicts for HF Qwen3.5 XML Jinja templates."""
out: List[Dict[str, Any]] = []
for m in messages:
mc = dict(m)
if "tool_calls" in mc and isinstance(mc["tool_calls"], list):
new_tcs = []
for tc in mc["tool_calls"]:
tc_c = dict(tc)
if "function" in tc_c and isinstance(tc_c["function"], dict):
fn_c = dict(tc_c["function"])
args = fn_c.get("arguments")
if isinstance(args, str):
try:
fn_c["arguments"] = json.loads(args)
except Exception:
pass
tc_c["function"] = fn_c
new_tcs.append(tc_c)
mc["tool_calls"] = new_tcs
out.append(mc)
return out
def _cast_xml_param(val: str, schema_type: Optional[str]) -> Any:
if schema_type == "integer":
try:
return int(val.strip())
except Exception:
return val
if schema_type == "number":
try:
return float(val.strip())
except Exception:
return val
if schema_type == "boolean":
low = val.strip().lower()
if low == "true":
return True
if low == "false":
return False
return val
if schema_type in ("array", "object"):
try:
return json.loads(val.strip())
except Exception:
return val
return val
def _parse_qwen_xml_tool_calls(text: str, tools: Optional[List[Dict[str, Any]]] = None) -> Tuple[str, List[Dict[str, Any]]]:
"""Parse Qwen3.5 native XML <tool_call><function=...> blocks into OpenAI-style dicts."""
if not text:
return "", []
prop_types: Dict[str, Dict[str, str]] = {}
for t in tools or []:
fn = t.get("function", t) if isinstance(t, dict) else {}
tname = str(fn.get("name", ""))
props = (fn.get("parameters") or {}).get("properties") or {}
if tname and isinstance(props, dict):
prop_types[tname] = {
str(pk): str(pv.get("type", "string")).lower()
for pk, pv in props.items()
if isinstance(pv, dict)
}
parsed: List[Dict[str, Any]] = []
# Match <tool_call>...<function=NAME>...</function>...</tool_call> or truncated tail
for m in re.finditer(r"<function=([A-Za-z0-9_.\-]+)>\s*(.*?)(?:</function>|</tool_call>|$)", text, flags=re.S):
name = m.group(1).strip()
inner = m.group(2)
args: Dict[str, Any] = {}
t_props = prop_types.get(name, {})
for pm in re.finditer(r"<parameter=([A-Za-z0-9_.\-]+)>\s*(.*?)\s*</parameter>", inner, flags=re.S):
pk = pm.group(1).strip()
pv = pm.group(2).strip()
args[pk] = _cast_xml_param(pv, t_props.get(pk))
if name:
parsed.append({"name": name, "arguments": args})
if not parsed:
for jm in re.finditer(r"<tool_call>\s*(\{.*?\})\s*</tool_call>", text, flags=re.S):
try:
obj = json.loads(jm.group(1))
if isinstance(obj, dict) and obj.get("name"):
parsed.append({"name": str(obj["name"]), "arguments": obj.get("arguments") or {}})
except Exception:
pass
cleaned = re.sub(r"<tool_call>.*?(?:</tool_call>|$)", "", text, flags=re.S).strip()
return cleaned, parsed
class DeepThoughtRouter(DynamicSubspaceRouter9S):
"""Dynamic 13-subspace Two-Phase Reasoning-Boundary LoRA router for DeepThought-STEM."""
def __init__(
self,
adapter_dir: Path,
model_name: str = "deepthought_9b",
max_norm_multiplier: float = 12.0,
use_tbench_subspaces: bool = True,
phase_split: bool = True,
stem_s12_scale: float = 1.55,
stem_s1_assist: float = 0.20,
fast_s1_scale: float = 0.90,
fast_s12_scale: float = 1.35,
stem_quant_s12_scale: float = 1.45,
stem_mcq_s13_scale: float = 1.20,
stem_concept_s12_scale: float = 1.00,
stem_concept_s13_scale: float = 0.40,
stem_math_s13_scale: float = 1.20,
fast_s13_scale: float = 0.50,
code_s1_scale: float = 0.95,
code_s12_scale: float = 0.95,
restraint_s1_scale: float = 1.45,
restraint_s12_scale: float = 0.95,
restraint_s2_scale: float = 1.45,
term_long_s1_scale: float = 1.65,
):
self.phase_split = bool(phase_split)
self.stem_s12_scale = float(stem_s12_scale)
self.stem_s1_assist = float(stem_s1_assist)
self.fast_s1_scale = float(fast_s1_scale)
self.fast_s12_scale = float(fast_s12_scale)
self.stem_quant_s12_scale = float(stem_quant_s12_scale)
self.stem_mcq_s13_scale = float(stem_mcq_s13_scale)
self.stem_concept_s12_scale = float(stem_concept_s12_scale)
self.stem_concept_s13_scale = float(stem_concept_s13_scale)
self.stem_math_s13_scale = float(stem_math_s13_scale)
self.fast_s13_scale = float(fast_s13_scale)
self.code_s1_scale = float(code_s1_scale)
self.code_s12_scale = float(code_s12_scale)
self.restraint_s1_scale = float(restraint_s1_scale)
self.restraint_s12_scale = float(restraint_s12_scale)
self.restraint_s2_scale = float(restraint_s2_scale)
self.term_long_s1_scale = float(term_long_s1_scale)
self.has_stem_subspace = False
self.has_post_concise_subspace = False
self._tokenizer = None
self._tokenizer_checked = False
super().__init__(
adapter_dir=adapter_dir,
model_name=model_name,
max_norm_multiplier=max_norm_multiplier,
use_tbench_subspaces=use_tbench_subspaces,
)
def _get_tokenizer(self):
if self._tokenizer_checked:
return self._tokenizer
self._tokenizer_checked = True
try:
from transformers import AutoTokenizer
except Exception:
return None
cands = [
self.adapter_dir.parent,
self.adapter_dir.parent.parent,
Path(r"C:\Users\jhjwi\.gemini\antigravity\scratch\qwen35-deepthought-stem-wsp\models\deepthought_9b"),
]
for c in cands:
if (c / "tokenizer_config.json").exists() or (c / "tokenizer.json").exists():
try:
self._tokenizer = AutoTokenizer.from_pretrained(str(c), trust_remote_code=True)
return self._tokenizer
except Exception:
pass
return None
def _load_metadata(self) -> None:
super()._load_metadata()
tpath = self.adapter_dir / "tbench_subspaces_manifest.json"
s12p = self.adapter_dir / "wsp_s12_stem_settle.gguf"
s13p = self.adapter_dir / "wsp_s13_post_concise.gguf"
if tpath.exists() and s12p.exists():
try:
tman = json.loads(tpath.read_text(encoding="utf-8"))
while len(self.subspace_frobs) < 11:
self.subspace_frobs.append(1.0)
if len(self.subspace_frobs) == 11:
self.subspace_frobs.append(float(tman.get("s12_frob", 1.0)))
self.has_stem_subspace = True
if s13p.exists() and "s13_frob" in tman:
if len(self.subspace_frobs) == 12:
self.subspace_frobs.append(float(tman.get("s13_frob", 1.0)))
self.has_post_concise_subspace = True
except Exception:
pass
@staticmethod
def _inspect_terminal_history(messages: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Track stateful tool turns, inspect streaks, real workspace mutations, and command loops."""
inspect_prefixes = (
"ls", "cat ", "head", "tail", "find ", "grep", "rg ", "git status",
"git log", "git branch", "git reflog", "git show", "git diff", "man ",
"df ", "tar -tf", "tar -tvf", "file ", "stat ", "pwd", "wc ", "which ",
"sha256sum", "xxd ", "hexdump ", "strings ", "od ", "nl ", "ldd ",
)
mutating_cues = (
"sed -i", "rm ", "mv ", "cp ", "chmod ", "chown ", "mkdir ", "touch ",
"tee ", "<<", "cat >", "shred ", "tar -x", "tar -c",
"git commit", "git checkout", "git merge", "git reset", "git add",
"git filter", "git rebase", "openssl ", "gpg ", "rencrypt ", "sqlite3 ",
"python3 -", "python -", "dos2unix",
)
cmd_list: List[str] = []
cmd_is_inspect: List[bool] = []
cmd_mutated_ws: List[bool] = []
for m in messages:
if str(m.get("role", "")).lower() != "assistant":
continue
for tc in (m.get("tool_calls") or []):
fn = tc.get("function", {}) if isinstance(tc, dict) else {}
if str(fn.get("name", "")).lower() != "exec":
continue
raw_args = fn.get("arguments", {})
if isinstance(raw_args, str):
try:
raw_args = json.loads(raw_args)
except Exception:
raw_args = {}
cmd = str(raw_args.get("command", "") or raw_args.get("cmd", "")).strip() if isinstance(raw_args, dict) else ""
if not cmd:
continue
cmd_list.append(cmd)
cl = cmd.lower()
stripped = re.sub(r"^\s*cd\s+[^;&|]+&&\s*", "", cl).strip()
# g++ stdin compilation or running `bash script.sh` without chmod does NOT mutate files on disk
non_mut_probe = (
("-x c++ -" in cl and "cat >" not in cl and "sed -i" not in cl and "tee " not in cl and not re.search(r"-o\s+(?!/dev/null|/tmp/)", cl))
or (stripped.startswith("bash ") and "chmod" not in cl and "sed " not in cl and not re.search(r"(?:^|\s)>>?\s*[^\s;&|]+", cl))
)
write_targets = re.findall(r"(?:^|\s)>>?\s*([^\s;&|]+)", cl)
has_ws_redir = any(not t.startswith(("/dev/null", "/tmp/")) for t in write_targets)
is_tmp_only_cat = bool(re.search(r"cat\b.*>\s*/tmp/", cl)) and not any(
k in cl for k in ("/app/", "/srv/", "/opt/", "/etc/", "sed -i", "chmod ", "openssl ", "gpg ")
)
mut_no_tmp = any(mc in cl for mc in mutating_cues) and not is_tmp_only_cat
is_mut = (not non_mut_probe) and (has_ws_redir or mut_no_tmp)
is_insp = (not is_mut) and any(stripped.startswith(p) for p in inspect_prefixes)
cmd_is_inspect.append(is_insp)
cmd_mutated_ws.append(is_mut)
tool_turn_count = sum(1 for m in messages if str(m.get("role", "")).lower() == "tool")
consec_inspect = 0
for flag in reversed(cmd_is_inspect):
if flag:
consec_inspect += 1
else:
break
repeated_loop = False
if len(cmd_list) >= 2:
last_norm = re.sub(r"\s+", " ", cmd_list[-1]).strip()
prev_norms = [re.sub(r"\s+", " ", c).strip() for c in cmd_list[:-1]]
if last_norm in prev_norms:
repeated_loop = True
elif len(last_norm) >= 24 and any(last_norm[:48] == pn[:48] for pn in prev_norms[-2:]):
repeated_loop = True
last_tool_txt = ""
for m in reversed(messages):
if str(m.get("role", "")).lower() == "tool":
last_tool_txt = _extract_text(m.get("content", ""))
break
lt_low = last_tool_txt.lower()
last_rc_nonzero = bool(re.search(r"\[exit_code=(?!0\b)-?\d+\]", lt_low))
last_tool_recov_err = last_rc_nonzero or any(
k in lt_low
for k in (
"tool call arguments were truncated",
"traceback (most recent call last)",
"command not found",
"no such file or directory",
"unrecognized option",
"invalid option",
"unknown flag",
"unknown option",
"syntax error",
"segmentation fault",
"inappropriate ioctl",
"no pinentry",
"operation cancelled",
)
)
return {
"tool_turn_count": tool_turn_count,
"consecutive_inspect_turns": consec_inspect,
"has_workspace_mutation": any(cmd_mutated_ws),
"repeated_cmd_loop": repeated_loop,
"last_tool_recov_err": last_tool_recov_err,
}
def _fit_scales(self, vals: List[float]) -> List[float]:
raw = list(vals[:9])
if len(self.subspace_frobs) >= 11:
raw.extend(vals[9:11])
if len(self.subspace_frobs) >= 12 or self.has_stem_subspace:
raw.append(vals[11] if len(vals) > 11 else 0.0)
if len(self.subspace_frobs) >= 13 or self.has_post_concise_subspace:
raw.append(vals[12] if len(vals) > 12 else 0.0)
weighted_norm = sum(abs(a) * f for a, f in zip(raw, self.subspace_frobs))
budget_cap = self.max_norm_multiplier * max(self.base_frob_ref, 1e-6)
if weighted_norm > budget_cap and weighted_norm > 0:
sc = budget_cap / weighted_norm
return [round(a * sc, 4) for a in raw]
return [round(a, 4) for a in raw]
def route(
self,
messages: List[Dict[str, Any]],
tools: Optional[List[Dict[str, Any]]] = None,
response_format: Optional[Dict[str, Any]] = None,
) -> DTRouteDecision:
t0 = time.perf_counter()
st = self._inspect_turn_state(messages, tools, response_format)
tb_st = self._inspect_terminal_history(messages)
st.update(tb_st)
proto = self._prototype_scores(st["last_user"] + ("\n" + st["tool_text"] if st["tool_text"] else ""))
u_low = st["last_user"].lower()
is_stem_mcq = ("multiple-choice" in u_low) or ("answer: (x)" in u_low)
is_math_comp = "simplified answer inside \\boxed{}" in u_low
is_stem_deep = (not st["has_tools"]) and (is_stem_mcq or is_math_comp)
is_math_boxed = ("\\boxed" in u_low) or is_stem_deep
code_cue = (not is_math_boxed) and (
bool(
re.search(
r"```|\bdef\s+\w+|\bclass\s+\w+|\bimplement\b|\balgorithm\b|\bpython\b|\btypescript\b|\brefactor\b|\bsort|\blinked list\b|\bpython function\b|\bwrite a function\b|\bcomplete the .*function\b",
u_low,
)
)
or proto[3] > 0.18
)
syco_or_factual_cue = proto[1] > 0.15 or any(
k in u_low
for k in (
"isn't it", "confirm that", "agree that", "always true",
"does chmod", "safe to run", "true or false", "why is",
"explain ", "capital city",
)
)
has_worker_tools = any(t in st["tool_names"] for t in ("exec", "web_search", "read_file", "write_file"))
non_action_code = any(
k in u_low
for k in ("write a concise python function", "explain the difference", "true or false", "confirm")
)
action_tool_cue = has_worker_tools and (not non_action_code) and (
proto[2] > proto[1] + 0.05
or any(
k in u_low
for k in (
"check ", "find ", "list ", "grep", "git ", "port ", "process",
"pid", "search ", "look up", "latest", "current ", "price ",
"fetch ", "http://", "https://", "read file", "read `",
"write file", "save the text", "edit file", "edit the /app/",
"/app/", "directory", "/var/", "logs", "run ", "execute ",
"command", "curl", "disk",
)
)
)
handoff_cue = st["unrecov_err"] or proto[5] > 0.22 or any(
k in u_low
for k in (
"escalate", "handoff", "hand off", "production database",
"drop table", "delete all", "api key", "secret key",
"iam secret", "password", "sudo ", "admin approval",
"cannot access", "beyond your permissions", "ambiguous", "out of scope",
"ssh", "sudoers", "private key", "/root/", "credential", "credentials", "secret",
)
)
# Indices 0..12 -> [s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13]
p1 = [0.0] * 13
p2 = [0.0] * 13
# 1. Stateful Terminal-Bench loop (WSL2) -- decoupled cleanly from S12/S13
if self.use_tbench_subspaces and self.has_tbench_subspaces and (st["is_stateful_terminal"] or st["tool_names"] == ["exec"]):
csv_or_etl = any(k in u_low for k in (".csv", ".parquet", "pandas"))
long_or_complex = (len(u_low) > 580) or any(k in u_low for k in ("polyglot", "sanitize", "grid"))
t2_p1_s1 = 1.25 if long_or_complex else 0.35
if not st["is_turn2_tool"]:
regime = "term_t1_action"
p1[0] = 0.55 if csv_or_etl else (self.term_long_s1_scale if long_or_complex else 0.30)
p2[9] = 0.50 if csv_or_etl else 0.85
p2[10] = 0.0 if csv_or_etl else (0.95 if long_or_complex else 0.85)
elif st["repeated_cmd_loop"]:
regime = "term_t2_loop_break"
p1[0] = 1.25 if long_or_complex else 0.40
p2[8] = 1.00 if csv_or_etl else 1.60
p2[9] = 0.0 if csv_or_etl else 0.60
p2[10] = 0.0 if csv_or_etl else 0.60
elif st["last_tool_recov_err"]:
regime = "term_t2_recovery"
p1[0] = t2_p1_s1
p2[8] = 1.00 if csv_or_etl else 1.20
p2[9] = 0.0 if csv_or_etl else 0.70
p2[10] = 0.0 if csv_or_etl else 0.70
elif (not csv_or_etl) and (st["consecutive_inspect_turns"] >= 2 or (st["tool_turn_count"] >= 2 and not st["has_workspace_mutation"])):
regime = "term_t2_force_mutate"
p1[0] = 1.25 if long_or_complex else 0.40
p2[9] = 1.25
p2[10] = 1.05
else:
regime = "term_t2_continue"
p1[0] = t2_p1_s1
p2[9] = 0.0 if csv_or_etl else (1.05 if st["is_inspect_cmd"] else 0.40)
p2[10] = 0.0 if csv_or_etl else (0.95 if st["is_inspect_cmd"] else 0.45)
# 2. Agent100 & STEM150 routing with clean P1 (<think>) vs P2 (post-</think>) split
elif st["is_turn2_tool"]:
if st["unrecov_err"]:
regime = "turn2_unrecoverable_handoff"
p1[0] = 0.50
p2[5] = 1.40
elif st["recov_err"]:
regime = "turn2_error_recovery"
p1[0] = 0.45
p2[8] = 0.95
p2[2] = 0.45
elif st["injection_cue"]:
regime = "turn2_injection_guard"
p1[0] = 0.35
p2[6] = 0.45
p2[1] = 0.65
elif st["grounding_cue"]:
regime = "turn2_grounded_synthesis"
p1[0] = 0.35
p2[7] = 0.90
p2[4] = 1.00 if st["wants_json"] else 0.0
p2[1] = 1.15
elif st["wants_json"]:
regime = "turn2_json_synthesis"
p1[0] = 0.35
p2[4] = 1.00
p2[1] = 0.70
if code_cue:
p2[3] = 0.50
else:
regime = "turn2_terminal_continue"
p1[0] = 0.50
p2[2] = 0.70 if st["is_inspect_cmd"] else 0.30
elif handoff_cue and (
st["has_handoff_tool"]
or (
not action_tool_cue
and any(
k in u_low
for k in (
"drop table", "delete all", "sudo ", "password", "api key", "iam secret",
"escalate", "handoff", "admin approval", "ssh", "sudoers", "private key",
"/root/", "credential", "credentials", "secret",
)
)
)
):
regime = "bounded_handoff"
p1[0] = 0.50
p2[5] = 3.40
p2[1] = 0.95 if not st["has_handoff_tool"] else 0.0
elif st["wants_json"] and not action_tool_cue:
regime = "structured_json"
p1[0] = 0.75
p2[4] = 0.95
p2[1] = 0.70 if st["has_tools"] else 0.35
if code_cue:
p2[3] = 0.50
elif is_stem_mcq and not st["has_tools"]:
is_stem_quant_mcq = any(
k in u_low
for k in (
"engineering question",
"chemistry question",
"physics question",
"math question",
"mathematics question",
"electrical_engineering",
"calculate",
"compute",
"find the",
"equation",
"integral",
"derivative",
"probability",
"voltage",
"current",
"molarity",
"velocity",
"acceleration",
"matrix",
"\\frac",
"$",
"^",
)
)
if is_stem_quant_mcq:
regime = "stem_mcq_quantitative"
p1[0] = 0.15
p1[11] = self.stem_quant_s12_scale
p2[12] = self.stem_mcq_s13_scale if self.has_post_concise_subspace else 0.0
else:
regime = "stem_mcq_conceptual"
p1[0] = 0.0
p1[11] = self.stem_concept_s12_scale
p2[12] = self.stem_concept_s13_scale if self.has_post_concise_subspace else 0.0
elif is_stem_deep:
regime = "stem_deep_settle"
p1[0] = self.stem_s1_assist
p1[11] = self.stem_s12_scale
p2[12] = self.stem_math_s13_scale if self.has_post_concise_subspace else 0.0
elif code_cue and not action_tool_cue:
regime = "mid_code_synthesis"
p1[0] = self.code_s1_scale
p1[11] = self.code_s12_scale if self.has_stem_subspace else 0.0
p2[3] = 0.65
p2[1] = 0.95 if has_worker_tools else 0.0
p2[12] = 0.0 # Keep code blocks uncut by S13
elif action_tool_cue:
regime = "tool_routing"
p1[0] = 1.25
p1[11] = 1.05 if self.has_stem_subspace else 0.0
p2[2] = 0.90
if code_cue:
p2[3] = 0.40
elif st["has_tools"] or syco_or_factual_cue:
regime = "restraint_pushback"
p1[0] = self.restraint_s1_scale
p1[11] = self.restraint_s12_scale if self.has_stem_subspace else 0.0
p2[1] = self.restraint_s2_scale
p2[12] = 0.85 if self.has_post_concise_subspace else 0.0
else:
regime = "general_fast_think"
p1[0] = self.fast_s1_scale
p1[11] = self.fast_s12_scale if self.has_stem_subspace else 0.0
p2[12] = self.fast_s13_scale if self.has_post_concise_subspace else 0.0
combined_vals = [
p2[i] if abs(p2[i]) > 1e-9 else p1[i]
for i in range(13)
]
p1_scales = self._fit_scales(p1)
p2_scales = self._fit_scales(p2)
comb_scales = self._fit_scales(combined_vals)
weighted_comb = sum(abs(a) * f for a, f in zip(combined_vals[:len(self.subspace_frobs)], self.subspace_frobs))
budget_cap = self.max_norm_multiplier * max(self.base_frob_ref, 1e-6)
budget_scale = min(1.0, budget_cap / weighted_comb) if weighted_comb > 0 else 1.0
recommend_fresh_kv = self.last_regime is not None and regime != self.last_regime
self.last_regime = regime
elapsed_ms = (time.perf_counter() - t0) * 1000.0
return DTRouteDecision(
scales=comb_scales,
phase1_scales=p1_scales,
phase2_scales=p2_scales,
regime=regime,
recommend_cache_prompt=not recommend_fresh_kv,
scores={DT_SUBSPACE_NAMES.get(i, f"s{i+1}"): round(proto[i], 4) for i in range(min(len(proto), 9))},
norm_budget_scale=round(budget_scale, 4),
router_ms=round(elapsed_ms, 3),
)
@staticmethod
def _post_json(url: str, body: Dict[str, Any], timeout: float = 240.0) -> Dict[str, Any]:
req = urllib.request.Request(
url,
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=timeout) as resp:
return json.loads(resp.read().decode("utf-8"))
def chat_completion(
self,
server_url: str,
messages: List[Dict[str, Any]],
tools: Optional[List[Dict[str, Any]]] = None,
response_format: Optional[Dict[str, Any]] = None,
max_tokens: int = 384,
temperature: float = 0.0,
force_scales: Optional[List[float]] = None,
cache_prompt_override: Optional[bool] = None,
) -> Dict[str, Any]:
tok = self._get_tokenizer() if (force_scales is None and response_format is None and self.phase_split) else None
if tok is None:
return super().chat_completion(
server_url=server_url,
messages=messages,
tools=tools,
response_format=response_format,
max_tokens=max_tokens,
temperature=temperature,
force_scales=force_scales,
cache_prompt_override=cache_prompt_override,
)
decision = self.route(messages, tools=tools, response_format=response_format)
p1_scales = decision.phase1_scales if decision.phase1_scales is not None else decision.scales
p2_scales = decision.phase2_scales if decision.phase2_scales is not None else decision.scales
base = server_url.rstrip("/")
if base.endswith("/v1"):
base = base[:-3]
rendered = tok.apply_chat_template(
_msgs_for_jinja(messages),
tools=(tools if tools else None),
tokenize=False,
add_generation_prompt=True,
)
# Phase 1: generate <think>...</think> using ONLY <think>-control subspaces
sw1_ms = self.push_scales(base, p1_scales)
t0 = time.perf_counter()
r1 = self._post_json(
f"{base}/completion",
{
"prompt": rendered,
"n_predict": int(max_tokens),
"temperature": float(temperature),
"seed": 42,
"stop": ["</think>", "<|im_end|>"],
"cache_prompt": False,
},
)
p1_ms = (time.perf_counter() - t0) * 1000.0
p1_text = r1.get("content") or ""
t1_tok = int(r1.get("tokens_predicted") or 0)
prompt_tok = int(r1.get("tokens_evaluated") or 0)
stop_type1 = r1.get("stop_type") or ""
stop_word1 = r1.get("stopping_word") or ""
sw2_ms = 0.0
p2_ms = 0.0
content_clean = ""
openai_tool_calls: Optional[List[Dict[str, Any]]] = None
comp_tok = t1_tok
if stop_word1 == "</think>":
rem = int(max_tokens) - t1_tok
if rem > 0:
sw2_ms = self.push_scales(base, p2_scales)
t1 = time.perf_counter()
r2 = self._post_json(
f"{base}/completion",
{
"prompt": rendered + p1_text + "</think>",
"n_predict": rem,
"temperature": float(temperature),
"seed": 42,
"stop": ["<|im_end|>", "<|endoftext|>"],
"cache_prompt": False,
},
)
p2_ms = (time.perf_counter() - t1) * 1000.0
p2_text = r2.get("content") or ""
t2_tok = int(r2.get("tokens_predicted") or 0)
stop_type2 = r2.get("stop_type") or ""
comp_tok = min(int(max_tokens), t1_tok + t2_tok)
if tools:
content_clean, parsed_tcs = _parse_qwen_xml_tool_calls(p2_text, tools=tools)
if parsed_tcs:
openai_tool_calls = [
{
"id": f"call_p2_{i}",
"type": "function",
"function": {
"name": tc["name"],
"arguments": json.dumps(tc["arguments"]),
},
}
for i, tc in enumerate(parsed_tcs)
]
else:
content_clean = p2_text.strip()
if stop_type2 == "limit":
finish_reason = "length"
elif openai_tool_calls:
finish_reason = "tool_calls"
else:
finish_reason = "stop"
else:
finish_reason = "length"
else:
if stop_type1 == "limit":
finish_reason = "length"
else:
if tools:
content_clean, parsed_tcs = _parse_qwen_xml_tool_calls(p1_text, tools=tools)
if parsed_tcs:
openai_tool_calls = [
{
"id": f"call_p1_{i}",
"type": "function",
"function": {
"name": tc["name"],
"arguments": json.dumps(tc["arguments"]),
},
}
for i, tc in enumerate(parsed_tcs)
]
finish_reason = "tool_calls" if openai_tool_calls else "stop"
return {
"decision": decision,
"scales_used": list(decision.scales),
"phase1_scales": p1_scales,
"phase2_scales": p2_scales,
"switch_ms": round(sw1_ms + sw2_ms, 3),
"infer_ms": round(p1_ms + sw1_ms + sw2_ms + p2_ms, 2),
"finish_reason": finish_reason,
"reasoning": p1_text.strip(),
"content": content_clean,
"tool_calls": openai_tool_calls,
"usage": {
"prompt_tokens": prompt_tok,
"completion_tokens": comp_tok,
},
}