xnnpack 4k: HuggingFaceTB/SmolLM2-360M-Instruct@a10cc1512eab

.gitattributes

@@ -50,3 +50,4 @@ mtk/mt6991/SmolLM2-360M-Instruct-neuropilot-a16w8-4k-chunk1of4.pte filter=lfs di
 mtk/mt6991/SmolLM2-360M-Instruct-neuropilot-a16w8-4k-chunk3of4.pte filter=lfs diff=lfs merge=lfs -text
 mtk/mt6991/SmolLM2-360M-Instruct-neuropilot-a16w8-4k-chunk4of4.pte filter=lfs diff=lfs merge=lfs -text
 mtk/mt6991/SmolLM2-360M-Instruct-neuropilot-a16w8-4k-chunk2of4.pte filter=lfs diff=lfs merge=lfs -text
+xnnpack/SmolLM2-360M-Instruct-fp32-4k.pte filter=lfs diff=lfs merge=lfs -text

README.md

@@ -26,7 +26,7 @@ Windows in this repo: XNNPACK (CPU) at 2k to 16k; Vulkan (GPU) at 2k to 32k; Med
 | Backend | Target | File | Window | Size | Smoke test |
 |---|---|---|---|---|---|
 | XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-360M-Instruct-8da4w-2k.pte`](xnnpack/SmolLM2-360M-Instruct-8da4w-2k.pte) | 2,048 tokens | 0.25 GB | passed ("Paris") |
-| XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-360M-Instruct-8da4w-4k.pte`](xnnpack/SmolLM2-360M-Instruct-8da4w-4k.pte) | 4,096 tokens | 0.25 GB | passed ("Paris") |
+| XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-360M-Instruct-fp32-4k.pte`](xnnpack/SmolLM2-360M-Instruct-fp32-4k.pte) | 4,096 tokens | 1.50 GB | passed ("Paris") |
 | XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-360M-Instruct-8da4w-8k.pte`](xnnpack/SmolLM2-360M-Instruct-8da4w-8k.pte) | 8,192 tokens | 0.26 GB | passed ("Paris") |
 | XNNPACK (CPU) | any arm64 | [`xnnpack/SmolLM2-360M-Instruct-8da4w-16k.pte`](xnnpack/SmolLM2-360M-Instruct-8da4w-16k.pte) | 16,384 tokens | 0.26 GB | passed ("Paris") |
 | Vulkan (GPU) | any arm64 | [`vulkan/SmolLM2-360M-Instruct-vulkan-8da4w-2k.pte`](vulkan/SmolLM2-360M-Instruct-vulkan-8da4w-2k.pte) | 2,048 tokens | 0.32 GB | structure checked (no host NPU runtime) |
@@ -61,7 +61,7 @@ Tokenizer: [`tokenizer.json`](tokenizer.json), copied unchanged from the source
 
 ## How it was made
 
-- XNNPACK (CPU) any arm64: ExecuTorch 1.4.0 `export_llm`: 8-bit dynamic activations and 4-bit weights in groups of 32, int8 per-channel embeddings, XNNPACK with extended ops, prefill chunk 2048, fp32 KV cache. Built by [run 1](https://github.com/ExperimentalMachines/executorch-model-exporter/actions/runs/34753839416).
+- XNNPACK (CPU) any arm64: ExecuTorch 1.4.0 `export_llm`: 8-bit dynamic activations and 4-bit weights in groups of 32, int8 per-channel embeddings, XNNPACK with extended ops, prefill chunk 2048, fp32 KV cache. Built by [run 1](https://github.com/ExperimentalMachines/execupack/actions/runs/37160993875), [run 2](https://github.com/ExperimentalMachines/executorch-model-exporter/actions/runs/34753839416).
 - Vulkan (GPU) any arm64: ExecuTorch 1.4.0 `export_llm`: 8-bit dynamic activations and 4-bit weights in groups of 32, int8 per-channel embeddings, the Vulkan delegate, prefill chunk 2048, fp32 KV cache. Built by [run 1](https://github.com/ExperimentalMachines/execupack/actions/runs/37157383005).
 - MediaTek NeuroPilot MT6991 (Dimensity 9400): ExecuTorch 1.4.0 MediaTek LLM export (`examples/mediatek`, `llama.py`): A16W8 (16-bit activations, 8-bit weights) calibrated on MediaTek's `alpaca.txt` prompts (9 of 9) in the qwen.json chat template, cut into 4 chunks, with a 128-token prompt graph and a one-token generation graph over a 2048-token cache, compiled with MediaTek NeuroPilot Express SDK (mtk_converter 8.13.0+public, mtk_neuron 8.2.23) for MT6991 (Dimensity 9400). Built by [run 1](https://github.com/ExperimentalMachines/execupack/actions/runs/37158194354).
 - MediaTek NeuroPilot MT6991 (Dimensity 9400): ExecuTorch 1.4.0 MediaTek LLM export (`examples/mediatek`, `llama.py`): A16W8 (16-bit activations, 8-bit weights) calibrated on MediaTek's `alpaca.txt` prompts (9 of 9) in the qwen.json chat template, cut into 4 chunks, with a 128-token prompt graph and a one-token generation graph over a 4096-token cache, compiled with MediaTek NeuroPilot Express SDK (mtk_converter 8.13.0+public, mtk_neuron 8.2.23) for MT6991 (Dimensity 9400). Built by [run 1](https://github.com/ExperimentalMachines/execupack/actions/runs/37158194354).

xnnpack/{SmolLM2-360M-Instruct-8da4w-4k.pte → SmolLM2-360M-Instruct-fp32-4k.pte}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bf1d26291299e07118ab46ac8bed99aac76eb30c50382780b6ee2a157ebd768
-size 253938816
+oid sha256:5d573a8f154193fd8a9d4454fce58c7dbd75edda17f25b74db401ab997627875
+size 1497383296

xnnpack/config.json

@@ -30,12 +30,12 @@
       }
     },
     {
-      "file": "SmolLM2-360M-Instruct-8da4w-4k.pte",
-      "size_bytes": 253938816,
-      "sha256": "4bf1d26291299e07118ab46ac8bed99aac76eb30c50382780b6ee2a157ebd768",
+      "file": "SmolLM2-360M-Instruct-fp32-4k.pte",
+      "size_bytes": 1497383296,
+      "sha256": "5d573a8f154193fd8a9d4454fce58c7dbd75edda17f25b74db401ab997627875",
       "context": 4096,
       "source_revision": "a10cc1512eabd3dde888204e902eca88bddb4951",
-      "quantization": "8da4w-g32, int8 embeddings",
+      "quantization": "fp32-g32, int8 embeddings, int4 codes rounded to nearest",
       "fits_phone_budget": true,
       "methods": {
         "get_max_context_len": 4096,

xnnpack/export-report-4k.json

@@ -38,12 +38,15 @@
       "use_hf_rope": false,
       "attention_qkv_bias": false
     },
-    "qmode": "8da4w",
+    "qmode": "fp32",
+    "export_llm_qmode": null,
     "group_size": 32,
     "embedding_quantize": "8,0",
+    "embedding_hqq": false,
+    "int4_codes": "round-to-nearest",
     "prefill_chunk": 2048,
     "kv_cache_dtype": "fp32",
-    "label": "8da4w-g32, int8 embeddings",
+    "label": "fp32-g32, int8 embeddings, int4 codes rounded to nearest",
     "description": "ExecuTorch 1.4.0 `export_llm`: 8-bit dynamic activations and 4-bit weights in groups of 32, int8 per-channel embeddings, XNNPACK with extended ops, prefill chunk 2048, fp32 KV cache."
   },
   "window": {
@@ -60,7 +63,7 @@
         "device_resident_bytes": 3471426488,
         "export_peak_bytes": 41179871488,
         "fits_device": true,
-        "fits_host": true
+        "fits_host": false
       },
       {
         "context": 16384,
@@ -98,27 +101,29 @@
   },
   "files": [
     {
-      "path": "xnnpack/SmolLM2-360M-Instruct-8da4w-4k.pte",
-      "bytes": 253938816,
-      "sha256": "4bf1d26291299e07118ab46ac8bed99aac76eb30c50382780b6ee2a157ebd768"
+      "path": "xnnpack/SmolLM2-360M-Instruct-fp32-4k.pte",
+      "bytes": 1497383296,
+      "sha256": "5d573a8f154193fd8a9d4454fce58c7dbd75edda17f25b74db401ab997627875"
     }
   ],
   "estimates": {
     "pte_bytes_estimate": 257418199,
-    "pte_bytes_actual": 253938816,
+    "pte_bytes_actual": 1497383296,
     "export_peak_bytes_estimate": 5008193792
   },
   "host": {
-    "nproc": 4,
-    "mem_total_bytes": 16766418944,
+    "nproc": 8,
+    "mem_total_bytes": 33229729792,
     "swap_total_bytes": 25769799680,
-    "cpu_model": "AMD EPYC 7763 64-Core Processor",
-    "peak_rss_convert_bytes": 1077006336,
-    "peak_rss_export_bytes": 3987619840,
+    "cpu_model": "AMD EPYC",
+    "peak_rss_convert_bytes": 1081634816,
+    "peak_rss_export_bytes": 6512148480,
     "peak_swap_used_bytes": 0,
-    "min_mem_available_bytes": 11627200512,
-    "export_seconds": 881.4,
-    "total_seconds": 892.7
+    "min_mem_available_bytes": 25611706368,
+    "peak_in_use_bytes": 7618023424,
+    "peak_in_use_at": "2026-10-03T23:25:14Z",
+    "export_seconds": 248.0,
+    "total_seconds": 256.5
   },
   "conversion": {
     "tensors": 291,
@@ -162,22 +167,136 @@
     "passed": true,
     "problems": [],
     "stats": {
-      "prefill_token_per_sec": 147.368,
-      "decode_token_per_sec": 71.4286,
+      "prefill_token_per_sec": 407.767,
+      "decode_token_per_sec": 31.25,
       "prompt_tokens": 42,
       "generated_tokens": 1,
-      "model_load_start_ms": 1789299113505,
-      "model_load_end_ms": 1789299114125,
-      "inference_start_ms": 1789299114125,
-      "inference_end_ms": 1789299114424,
-      "prompt_eval_end_ms": 1789299114410,
-      "first_token_ms": 1789299114410,
+      "model_load_start_ms": 1791070124593,
+      "model_load_end_ms": 1791070125653,
+      "inference_start_ms": 1791070125653,
+      "inference_end_ms": 1791070125788,
+      "prompt_eval_end_ms": 1791070125756,
+      "first_token_ms": 1791070125756,
       "aggregate_sampling_time_ms": 0,
       "SCALING_FACTOR_UNITS_PER_SECOND": 1000
     }
   },
   "run": {
-    "url": "https://github.com/ExperimentalMachines/executorch-model-exporter/actions/runs/34753839416",
-    "id": "34753839416"
+    "url": "https://github.com/ExperimentalMachines/execupack/actions/runs/37160993875",
+    "id": "37160993875"
+  },
+  "measuring_gate": {
+    "kind": "measuring_gate",
+    "passed": true,
+    "prefill_tok_per_sec": 407.77,
+    "decode_tok_per_sec": 31.25,
+    "prompt_tokens": 42,
+    "generated_tokens": 1
+  },
+  "gate": {
+    "passed": true,
+    "problems": [],
+    "graded": 2,
+    "fp32_mean": 0.5698,
+    "export_mean": 0.5673,
+    "agreed": 2,
+    "shortfall": 0.0025,
+    "quiet_rows_answered_like_fp32": 4,
+    "measures_tool_calling": true,
+    "quiet_rows": 4,
+    "fp32_undecided_rows": [
+      "What is Concord the capital of?",
+      "What is Augusta the capital of?",
+      "Who is the author of Mercy?",
+      "Who is the author of Landfall?",
+      "Who is the author of Quicksand?",
+      "Who is the author of Nightfall?",
+      "In what country is Ostrava?",
+      "In what country is Gdynia?",
+      "In what country is Rijeka?",
+      "In what country is Linkoping?"
+    ],
+    "rows": {
+      "What is Concord the capital of?": {
+        "fp32": 0.2562,
+        "export": 0.253,
+        "kind": "search"
+      },
+      "What is Providence the capital of?": {
+        "fp32": 0.6218,
+        "export": 0.6241,
+        "kind": "search"
+      },
+      "What is Augusta the capital of?": {
+        "fp32": 0.0992,
+        "export": 0.0976,
+        "kind": "search"
+      },
+      "What is Nassau the capital of?": {
+        "fp32": 0.5179,
+        "export": 0.5106,
+        "kind": "search"
+      },
+      "Who is the author of Mercy?": {
+        "fp32": 0.134,
+        "export": 0.1291,
+        "kind": "search"
+      },
+      "Who is the author of Landfall?": {
+        "fp32": 0.1496,
+        "export": 0.1439,
+        "kind": "search"
+      },
+      "Who is the author of Quicksand?": {
+        "fp32": 0.1766,
+        "export": 0.1726,
+        "kind": "search"
+      },
+      "Who is the author of Nightfall?": {
+        "fp32": 0.1597,
+        "export": 0.1565,
+        "kind": "search"
+      },
+      "In what country is Ostrava?": {
+        "fp32": 0.3497,
+        "export": 0.335,
+        "kind": "search"
+      },
+      "In what country is Gdynia?": {
+        "fp32": 0.3652,
+        "export": 0.3542,
+        "kind": "search"
+      },
+      "In what country is Rijeka?": {
+        "fp32": 0.4028,
+        "export": 0.3914,
+        "kind": "search"
+      },
+      "In what country is Linkoping?": {
+        "fp32": 0.2624,
+        "export": 0.2553,
+        "kind": "search"
+      },
+      "What is the capital of Norway?": {
+        "fp32": 0.4101,
+        "export": 0.4042,
+        "kind": "quiet"
+      },
+      "What is 8 times 7?": {
+        "fp32": 0.4963,
+        "export": 0.4935,
+        "kind": "quiet"
+      },
+      "Who wrote Pride and Prejudice?": {
+        "fp32": 0.1944,
+        "export": 0.1993,
+        "kind": "quiet"
+      },
+      "What is the chemical symbol for iron?": {
+        "fp32": 0.3696,
+        "export": 0.3826,
+        "kind": "quiet"
+      }
+    }
   }
 }