{ "date": "2026-10-11", "release": "20261008-39663", "dataset": { "name": "TunSwitch test_cs + test_to", "clips": 644, "seconds": 3010.7168125, "words": 6990, "characters": 30060, "codeSwitchedClips": 300, "tunisianOnlyClips": 344, "manifestSha256": "483b3d4337707721b71546521600c93d2afea490cf9ff3475bba3bdf3a24cf1c", "blindHoldout": false }, "models": [ { "id": "fennec", "name": "Fennec", "provider": "Baarcha", "configuration": "20261008-39663 · frozen CTC decoder", "method": "FP32, TF32 off; linear_union_prefix decoder, beam 100, alpha 0.3, beta 0.5, old/new LM weights 0.25/0.75.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 1684, "substitutions": 1144, "deletions": 277, "insertions": 263, "chars": 30060, "char_errors": 2631, "wer": 24.09155937052933, "cer": 8.75249500998004 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 1127, "substitutions": 735, "deletions": 179, "insertions": 213, "chars": 18350, "char_errors": 1771, "wer": 26.492712740949695, "cer": 9.651226158038147 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 557, "substitutions": 409, "deletions": 98, "insertions": 50, "chars": 11710, "char_errors": 860, "wer": 20.358187134502923, "cer": 7.344150298889838 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 0, "http_retried_clips": 0 } }, { "id": "linto", "name": "LinTO Tunisian", "provider": "LinTO", "configuration": "Vosk · CPU", "method": "Local Vosk Tunisian model; standard 4,000-byte PCM chunks.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 2207, "substitutions": 1361, "deletions": 680, "insertions": 166, "chars": 30060, "char_errors": 5362, "wer": 31.573676680972817, "cer": 17.837658017298736 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 1341, "substitutions": 875, "deletions": 350, "insertions": 116, "chars": 18350, "char_errors": 3248, "wer": 31.52327221438646, "cer": 17.70027247956403 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 866, "substitutions": 486, "deletions": 330, "insertions": 50, "chars": 11710, "char_errors": 2114, "wer": 31.65204678362573, "cer": 18.052946199829204 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 12, "http_retried_clips": 0 } }, { "id": "oddadmix", "name": "Whisper Tunisian CodeSwitch", "provider": "Oddadmix", "configuration": "Whisper v3 · greedy / Arabic", "method": "FP32, greedy decoding, language=ar, maximum 440 new tokens.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 2381, "substitutions": 1509, "deletions": 240, "insertions": 632, "chars": 30060, "char_errors": 4213, "wer": 34.062947067238916, "cer": 14.015302727877579 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 1432, "substitutions": 887, "deletions": 130, "insertions": 415, "chars": 18350, "char_errors": 1965, "wer": 33.66243535496004, "cer": 10.708446866485014 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 949, "substitutions": 622, "deletions": 110, "insertions": 217, "chars": 11710, "char_errors": 2248, "wer": 34.68567251461988, "cer": 19.19726729291204 } }, "reliability": { "possible_truncations": 2, "service_failures": 0, "empty_transcripts": 0, "http_retried_clips": 0 } }, { "id": "mai", "name": "MAI-Transcribe 2", "provider": "Microsoft", "configuration": "Verbatim · automatic language", "method": "microsoft/mai-transcribe-2 via OpenRouter; Azure enhanced mode, verbatim transcription, automatic language.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 2952, "substitutions": 2101, "deletions": 435, "insertions": 416, "chars": 30060, "char_errors": 4904, "wer": 42.23175965665236, "cer": 16.31403858948769 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 1845, "substitutions": 1293, "deletions": 229, "insertions": 323, "chars": 18350, "char_errors": 3020, "wer": 43.37094499294781, "cer": 16.457765667574932 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1107, "substitutions": 808, "deletions": 206, "insertions": 93, "chars": 11710, "char_errors": 1884, "wer": 40.46052631578947, "cer": 16.088812980358668 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 11, "http_retried_clips": 0 } }, { "id": "openai", "name": "GPT-Transcribe", "provider": "OpenAI", "configuration": "Automatic language · no prompt", "method": "gpt-transcribe through the OpenAI transcription API; no language hint or prompt.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 3406, "substitutions": 2397, "deletions": 862, "insertions": 147, "chars": 30060, "char_errors": 7538, "wer": 48.726752503576535, "cer": 25.07651363938789 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 2176, "substitutions": 1501, "deletions": 583, "insertions": 92, "chars": 18350, "char_errors": 5279, "wer": 51.151857075693464, "cer": 28.768392370572208 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1230, "substitutions": 896, "deletions": 279, "insertions": 55, "chars": 11710, "char_errors": 2259, "wer": 44.95614035087719, "cer": 19.291204099060632 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 11, "http_retried_clips": 0 } }, { "id": "gemini", "name": "Gemini 3.5 Transcribe", "provider": "Google", "configuration": "Automatic language · no prompt", "method": "google/gemini-3.5-transcribe via OpenRouter; automatic language, no prompt.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 4413, "substitutions": 2430, "deletions": 1757, "insertions": 226, "chars": 30060, "char_errors": 12559, "wer": 63.13304721030043, "cer": 41.77977378576181 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 2880, "substitutions": 1320, "deletions": 1427, "insertions": 133, "chars": 18350, "char_errors": 9437, "wer": 67.70098730606487, "cer": 51.427792915531334 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1533, "substitutions": 1110, "deletions": 330, "insertions": 93, "chars": 11710, "char_errors": 3122, "wer": 56.030701754385966, "cer": 26.660973526900086 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 17, "http_retried_clips": 0 } }, { "id": "qwen", "name": "Qwen3-ASR 1.7B", "provider": "Qwen", "configuration": "BF16 · batch 12", "method": "Stock Qwen3-ASR-1.7B, BF16, SDPA, left padding, batch 12, greedy decoding, maximum 512 new tokens. Automatic language.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 4795, "substitutions": 3732, "deletions": 738, "insertions": 325, "chars": 30060, "char_errors": 10978, "wer": 68.59799713876967, "cer": 36.52029274783766 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 3149, "substitutions": 2448, "deletions": 438, "insertions": 263, "chars": 18350, "char_errors": 8266, "wer": 74.02444757874942, "cer": 45.04632152588556 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1646, "substitutions": 1284, "deletions": 300, "insertions": 62, "chars": 11710, "char_errors": 2712, "wer": 60.16081871345029, "cer": 23.159692570452606 } }, "reliability": { "possible_truncations": 3, "service_failures": 0, "empty_transcripts": 11, "http_retried_clips": 0 } }, { "id": "whisper", "name": "Whisper large-v3", "provider": "OpenAI · local", "configuration": "Greedy decoding / Arabic", "method": "Stock Whisper large-v3, FP32, greedy decoding, language=ar, maximum 440 new tokens.", "diagnostic": false, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 5909, "substitutions": 3641, "deletions": 1233, "insertions": 1035, "chars": 30060, "char_errors": 16973, "wer": 84.53505007153076, "cer": 56.463739188290084 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 4268, "substitutions": 2459, "deletions": 987, "insertions": 822, "chars": 18350, "char_errors": 14037, "wer": 100.32910202162671, "cer": 76.49591280653951 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1641, "substitutions": 1182, "deletions": 246, "insertions": 213, "chars": 11710, "char_errors": 2936, "wer": 59.978070175438596, "cer": 25.07258753202391 } }, "reliability": { "possible_truncations": 7, "service_failures": 0, "empty_transcripts": 0, "http_retried_clips": 0 } }, { "id": "openai_hinted", "name": "GPT-Transcribe with hints", "provider": "OpenAI", "configuration": "Tounsi context + ar/fr/en", "method": "Same model, with generic Tunisian context and Arabic/French/English language hints. No reference transcript supplied.", "diagnostic": true, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 3071, "substitutions": 2171, "deletions": 761, "insertions": 139, "chars": 30060, "char_errors": 6071, "wer": 43.934191702432045, "cer": 20.196274118429805 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 2007, "substitutions": 1397, "deletions": 519, "insertions": 91, "chars": 18350, "char_errors": 4506, "wer": 47.179125528913964, "cer": 24.555858310626704 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1064, "substitutions": 774, "deletions": 242, "insertions": 48, "chars": 11710, "char_errors": 1565, "wer": 38.888888888888886, "cer": 13.36464560204953 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 12, "http_retried_clips": 0 } }, { "id": "gemini_ar", "name": "Gemini with Arabic hint", "provider": "Google", "configuration": "language=ar", "method": "Same Gemini model, with language=ar. One runaway output inserted 2,369 words against a 10-word reference; it remains in the score. Two failed requests succeeded on retry.", "diagnostic": true, "scores": { "all": { "successful": 644, "expected": 644, "words": 6990, "errors": 6512, "substitutions": 2155, "deletions": 1831, "insertions": 2526, "chars": 30060, "char_errors": 14134, "wer": 93.16165951359085, "cer": 47.01929474384564 }, "test_cs": { "successful": 300, "expected": 300, "words": 4254, "errors": 5262, "substitutions": 1250, "deletions": 1564, "insertions": 2448, "chars": 18350, "char_errors": 11955, "wer": 123.6953455571227, "cer": 65.14986376021798 }, "test_to": { "successful": 344, "expected": 344, "words": 2736, "errors": 1250, "substitutions": 905, "deletions": 267, "insertions": 78, "chars": 11710, "char_errors": 2179, "wer": 45.687134502923975, "cer": 18.60802732707088 } }, "reliability": { "possible_truncations": 0, "service_failures": 0, "empty_transcripts": 13, "http_retried_clips": 2 } } ], "audit": { "scores": [ { "stage": "original", "subset": "all", "words": 6990, "errors": 1684, "substitutions": 1144, "deletions": 277, "insertions": 263, "chars": 30060, "char_errors": 2631, "wer": 24.09155937052933, "cer": 8.75249500998004 }, { "stage": "original", "subset": "test_cs", "words": 4254, "errors": 1127, "substitutions": 735, "deletions": 179, "insertions": 213, "chars": 18350, "char_errors": 1771, "wer": 26.492712740949695, "cer": 9.651226158038147 }, { "stage": "original", "subset": "test_to", "words": 2736, "errors": 557, "substitutions": 409, "deletions": 98, "insertions": 50, "chars": 11710, "char_errors": 860, "wer": 20.358187134502923, "cer": 7.344150298889838 }, { "stage": "spacing", "subset": "all", "words": 6990, "errors": 1446, "substitutions": 1006, "deletions": 218, "insertions": 222, "chars": 30060, "char_errors": 2631, "wer": 20.686695278969957, "cer": 8.75249500998004 }, { "stage": "spacing", "subset": "test_cs", "words": 4254, "errors": 952, "substitutions": 629, "deletions": 127, "insertions": 196, "chars": 18350, "char_errors": 1771, "wer": 22.37893747061589, "cer": 9.651226158038147 }, { "stage": "spacing", "subset": "test_to", "words": 2736, "errors": 494, "substitutions": 377, "deletions": 91, "insertions": 26, "chars": 11710, "char_errors": 860, "wer": 18.055555555555557, "cer": 7.344150298889838 }, { "stage": "spelling", "subset": "all", "words": 6990, "errors": 1197, "substitutions": 757, "deletions": 218, "insertions": 222, "chars": 30060, "char_errors": 2380, "wer": 17.124463519313306, "cer": 7.917498336660013 }, { "stage": "spelling", "subset": "test_cs", "words": 4254, "errors": 814, "substitutions": 491, "deletions": 127, "insertions": 196, "chars": 18350, "char_errors": 1636, "wer": 19.13493182886695, "cer": 8.915531335149863 }, { "stage": "spelling", "subset": "test_to", "words": 2736, "errors": 383, "substitutions": 266, "deletions": 91, "insertions": 26, "chars": 11710, "char_errors": 744, "wer": 13.998538011695906, "cer": 6.353543979504697 }, { "stage": "script", "subset": "all", "words": 6990, "errors": 1116, "substitutions": 691, "deletions": 201, "insertions": 224, "chars": 30060, "char_errors": 2051, "wer": 15.965665236051501, "cer": 6.823020625415835 }, { "stage": "script", "subset": "test_cs", "words": 4254, "errors": 733, "substitutions": 425, "deletions": 110, "insertions": 198, "chars": 18350, "char_errors": 1307, "wer": 17.230841560883874, "cer": 7.122615803814714 }, { "stage": "script", "subset": "test_to", "words": 2736, "errors": 383, "substitutions": 266, "deletions": 91, "insertions": 26, "chars": 11710, "char_errors": 744, "wer": 13.998538011695906, "cer": 6.353543979504697 }, { "stage": "colloquial", "subset": "all", "words": 6990, "errors": 992, "substitutions": 581, "deletions": 191, "insertions": 220, "chars": 30060, "char_errors": 1936, "wer": 14.19170243204578, "cer": 6.44045242847638 }, { "stage": "colloquial", "subset": "test_cs", "words": 4254, "errors": 658, "substitutions": 359, "deletions": 105, "insertions": 194, "chars": 18350, "char_errors": 1241, "wer": 15.467795016455101, "cer": 6.762942779291553 }, { "stage": "colloquial", "subset": "test_to", "words": 2736, "errors": 334, "substitutions": 222, "deletions": 86, "insertions": 26, "chars": 11710, "char_errors": 695, "wer": 12.207602339181287, "cer": 5.935098206660974 } ], "incrementalEditsForgiven": { "spacing": 238, "spelling": 249, "script": 81, "colloquial": 124 }, "residualCategories": { "one_character_substitution": 249, "delete": 191, "other_substitution": 332, "insert": 220 }, "possibleFillerInsertions": 68, "audioAdjudicated": false }, "timing": { "model": "fennec", "clips": 644, "medianSeconds": 0.03380681108683348, "p95Seconds": 0.08045580415055156, "realTimeFactor": 0.00867078388859425, "hardware": "NVIDIA B200", "batchSize": 1, "measurement": "Local per-clip transcription with the model already loaded, including audio handling and decoding. All clips included; no warm-up clips excluded.", "excludes": [ "Model loading", "Upload and network time", "Gateway overhead" ], "limitations": "Shared hardware, sequential short clips. Not an end-to-end service latency or a concurrency test." }, "normalization": "NFKC; lowercase; Arabic vowel marks/tatweel removed; alef variants unified; punctuation/symbols/control characters spaced except internal apostrophes and decimal/group separators. French accents, negation, numbers, and word boundaries retained. No lexical substitutions. Raw: NFC + whitespace only. CER excludes whitespace. Corpus WER is sum(edit errors)/sum(reference words).", "limitations": [ "Public sets repeatedly inspected during model development. No blind generalization claim.", "No verified speaker IDs; confidence intervals resample clips only.", "Service failures excluded from conditional quality scores and reported separately; compare complete sets or matched pairs.", "Local/API latency includes different hardware/network paths; concurrent shared host, not isolated performance testing.", "No human annotation of named entities, negation, or numbers in this run; WER/CER do not establish semantic accuracy.", "Default Whisper baselines use language=ar following the Tunisian model card; other default arms use automatic routing/detection with no prompt or reference hints. Exploratory hint arms are reported separately.", "Commercial model IDs are recorded with timestamps and request IDs, but no immutable provider snapshot or internal weights are exposed by these transcription responses.", "Gemini with an Arabic hint produced one runaway output on test_cs:51_86_2: 2,369 inserted words against a 10-word reference. This output is retained and substantially worsens its corpus WER.", "Qwen uses batch size 12, BF16, SDPA, and left padding; latency is amortized batch time. Its single-clip smoke differs on 2/3 texts; original partial single-clip results are archived, not mixed into scores.", "Language-hint arms are exploratory diagnostics added after inspecting baseline development outputs/early results; retain baseline and diagnostic scores, with no claim of a preregistered hint comparison.", "Generation reaching the token cap remains in scores and is reported; WER can exceed 100% when a model inserts many words." ], "auditLimitations": [ "Residual edits are unresolved reference disagreements, not a count of verified wrong spoken words. Audio and a Tunisian-speaking human adjudicator are needed to establish that count.", "Spelling and script equivalences were selected after inspecting Fennec differences; this is exploratory and model-specific. Freeze and apply the same policy to all model outputs before using it for a revised comparison.", "The broad colloquial policy is more interpretive; report it separately from spelling/script normalization.", "Close spelling alone is never forgiven. Negation, person, gender, numbers, insertions, and missing phrases remain counted unless the explicit allowed surface form is equivalent.", "The reference itself contains apparent mistakes, including the token صذةةةةةةةةةةة. This audit retains that token and other unresolved reference problems.", "The original suite was inspected during Fennec development. Adjusting transcription scoring does not make it a blind test.", "Counts are edit operations, not distinct wrong reference words; insertions count even though they have no corresponding reference word." ], "sourceSha256": { "results.json": "bd130ab28e172c690d4ece1ccfdf007f42c164d9b239028719a543e0bc2c7932", "audit_summary.json": "5848020fdff8732d8cfd51a6702b000e66fc8531eed5f760bdfb98bd83bf5038" } }