{
  "name": "TextKit Tokenizer Comparison 2026",
  "corpus_version": "1.0",
  "generated": "2026-06-10",
  "license": "CC BY 4.0 — cite textkit.tech when reusing",
  "methodology": {
    "openai_encodings": "tiktoken 0.13.0 (exact)",
    "claude": "official /v1/messages/count_tokens endpoint, envelope-calibrated (measured envelopes: {'claude-opus-4-8': 6, 'claude-sonnet-4-6': 7, 'claude-haiku-4-5': 7}; absolute counts ±1 token)",
    "claude_family_counts_identical": false,
    "pricing_usd_per_1M_input_tokens": {
      "gpt-5": 1.25,
      "gpt-5-mini": 0.25,
      "gpt-4o": 2.5,
      "claude-opus-4-8": 5.0,
      "claude-sonnet-4-6": 3.0,
      "claude-haiku-4-5": 1.0
    }
  },
  "measurements": [
    {
      "id": "L1-en",
      "label": "English prose",
      "category": "language-parallel",
      "script": "latin",
      "chars": 572,
      "words": 94,
      "o200k_base": 110,
      "cl100k_base": 110,
      "p50k_base": 112,
      "r50k_base": 112,
      "claude-opus-4-8": 177,
      "claude-sonnet-4-6": 116,
      "claude-haiku-4-5": 116,
      "tok_per_word_o200k": 1.17,
      "tok_per_100chars_o200k": 19.23,
      "vs_english_o200k": 1.0,
      "tok_per_word_cl100k": 1.17,
      "tok_per_100chars_cl100k": 19.23,
      "vs_english_cl100k": 1.0,
      "tok_per_word_opus-4-8": 1.883,
      "tok_per_100chars_opus-4-8": 30.94,
      "vs_english_opus-4-8": 1.0
    },
    {
      "id": "L2-es",
      "label": "Spanish prose",
      "category": "language-parallel",
      "script": "latin",
      "chars": 648,
      "words": 107,
      "o200k_base": 143,
      "cl100k_base": 172,
      "p50k_base": 222,
      "r50k_base": 222,
      "claude-opus-4-8": 256,
      "claude-sonnet-4-6": 184,
      "claude-haiku-4-5": 184,
      "tok_per_word_o200k": 1.336,
      "tok_per_100chars_o200k": 22.07,
      "vs_english_o200k": 1.3,
      "tok_per_word_cl100k": 1.607,
      "tok_per_100chars_cl100k": 26.54,
      "vs_english_cl100k": 1.564,
      "tok_per_word_opus-4-8": 2.393,
      "tok_per_100chars_opus-4-8": 39.51,
      "vs_english_opus-4-8": 1.446
    },
    {
      "id": "L3-pt",
      "label": "Portuguese prose",
      "category": "language-parallel",
      "script": "latin",
      "chars": 612,
      "words": 102,
      "o200k_base": 137,
      "cl100k_base": 176,
      "p50k_base": 225,
      "r50k_base": 225,
      "claude-opus-4-8": 241,
      "claude-sonnet-4-6": 188,
      "claude-haiku-4-5": 188,
      "tok_per_word_o200k": 1.343,
      "tok_per_100chars_o200k": 22.39,
      "vs_english_o200k": 1.245,
      "tok_per_word_cl100k": 1.725,
      "tok_per_100chars_cl100k": 28.76,
      "vs_english_cl100k": 1.6,
      "tok_per_word_opus-4-8": 2.363,
      "tok_per_100chars_opus-4-8": 39.38,
      "vs_english_opus-4-8": 1.362
    },
    {
      "id": "L4-fr",
      "label": "French prose",
      "category": "language-parallel",
      "script": "latin",
      "chars": 686,
      "words": 109,
      "o200k_base": 153,
      "cl100k_base": 194,
      "p50k_base": 234,
      "r50k_base": 234,
      "claude-opus-4-8": 275,
      "claude-sonnet-4-6": 207,
      "claude-haiku-4-5": 207,
      "tok_per_word_o200k": 1.404,
      "tok_per_100chars_o200k": 22.3,
      "vs_english_o200k": 1.391,
      "tok_per_word_cl100k": 1.78,
      "tok_per_100chars_cl100k": 28.28,
      "vs_english_cl100k": 1.764,
      "tok_per_word_opus-4-8": 2.523,
      "tok_per_100chars_opus-4-8": 40.09,
      "vs_english_opus-4-8": 1.554
    },
    {
      "id": "L5-de",
      "label": "German prose",
      "category": "language-parallel",
      "script": "latin",
      "chars": 665,
      "words": 93,
      "o200k_base": 159,
      "cl100k_base": 203,
      "p50k_base": 273,
      "r50k_base": 273,
      "claude-opus-4-8": 324,
      "claude-sonnet-4-6": 245,
      "claude-haiku-4-5": 245,
      "tok_per_word_o200k": 1.71,
      "tok_per_100chars_o200k": 23.91,
      "vs_english_o200k": 1.445,
      "tok_per_word_cl100k": 2.183,
      "tok_per_100chars_cl100k": 30.53,
      "vs_english_cl100k": 1.845,
      "tok_per_word_opus-4-8": 3.484,
      "tok_per_100chars_opus-4-8": 48.72,
      "vs_english_opus-4-8": 1.831
    },
    {
      "id": "L6-zh",
      "label": "Chinese prose (Simplified)",
      "category": "language-parallel",
      "script": "cjk",
      "chars": 191,
      "words": null,
      "o200k_base": 159,
      "cl100k_base": 223,
      "p50k_base": 389,
      "r50k_base": 389,
      "claude-opus-4-8": 216,
      "claude-sonnet-4-6": 217,
      "claude-haiku-4-5": 217,
      "tok_per_100chars_o200k": 83.25,
      "vs_english_o200k": 1.445,
      "tok_per_100chars_cl100k": 116.75,
      "vs_english_cl100k": 2.027,
      "tok_per_100chars_opus-4-8": 113.09,
      "vs_english_opus-4-8": 1.22
    },
    {
      "id": "L7-ja",
      "label": "Japanese prose",
      "category": "language-parallel",
      "script": "cjk",
      "chars": 234,
      "words": null,
      "o200k_base": 205,
      "cl100k_base": 268,
      "p50k_base": 373,
      "r50k_base": 373,
      "claude-opus-4-8": 240,
      "claude-sonnet-4-6": 241,
      "claude-haiku-4-5": 241,
      "tok_per_100chars_o200k": 87.61,
      "vs_english_o200k": 1.864,
      "tok_per_100chars_cl100k": 114.53,
      "vs_english_cl100k": 2.436,
      "tok_per_100chars_opus-4-8": 102.56,
      "vs_english_opus-4-8": 1.356
    },
    {
      "id": "C1-python",
      "label": "Python code",
      "category": "code",
      "script": "latin",
      "chars": 667,
      "words": 77,
      "o200k_base": 167,
      "cl100k_base": 167,
      "p50k_base": 219,
      "r50k_base": 271,
      "claude-opus-4-8": 254,
      "claude-sonnet-4-6": 208,
      "claude-haiku-4-5": 208,
      "tok_per_word_o200k": 2.169,
      "tok_per_100chars_o200k": 25.04,
      "tok_per_word_cl100k": 2.169,
      "tok_per_100chars_cl100k": 25.04,
      "tok_per_word_opus-4-8": 3.299,
      "tok_per_100chars_opus-4-8": 38.08
    },
    {
      "id": "C2-javascript",
      "label": "JavaScript code",
      "category": "code",
      "script": "latin",
      "chars": 636,
      "words": 66,
      "o200k_base": 149,
      "cl100k_base": 140,
      "p50k_base": 209,
      "r50k_base": 213,
      "claude-opus-4-8": 242,
      "claude-sonnet-4-6": 190,
      "claude-haiku-4-5": 190,
      "tok_per_word_o200k": 2.258,
      "tok_per_100chars_o200k": 23.43,
      "tok_per_word_cl100k": 2.121,
      "tok_per_100chars_cl100k": 22.01,
      "tok_per_word_opus-4-8": 3.667,
      "tok_per_100chars_opus-4-8": 38.05
    },
    {
      "id": "S1-json",
      "label": "JSON data",
      "category": "structured",
      "script": "latin",
      "chars": 521,
      "words": 60,
      "o200k_base": 214,
      "cl100k_base": 213,
      "p50k_base": 242,
      "r50k_base": 254,
      "claude-opus-4-8": 284,
      "claude-sonnet-4-6": 249,
      "claude-haiku-4-5": 249,
      "tok_per_word_o200k": 3.567,
      "tok_per_100chars_o200k": 41.07,
      "tok_per_word_cl100k": 3.55,
      "tok_per_100chars_cl100k": 40.88,
      "tok_per_word_opus-4-8": 4.733,
      "tok_per_100chars_opus-4-8": 54.51
    },
    {
      "id": "S2-markdown",
      "label": "Markdown document",
      "category": "structured",
      "script": "latin",
      "chars": 639,
      "words": 104,
      "o200k_base": 162,
      "cl100k_base": 161,
      "p50k_base": 187,
      "r50k_base": 187,
      "claude-opus-4-8": 239,
      "claude-sonnet-4-6": 175,
      "claude-haiku-4-5": 175,
      "tok_per_word_o200k": 1.558,
      "tok_per_100chars_o200k": 25.35,
      "tok_per_word_cl100k": 1.548,
      "tok_per_100chars_cl100k": 25.2,
      "tok_per_word_opus-4-8": 2.298,
      "tok_per_100chars_opus-4-8": 37.4
    },
    {
      "id": "S3-social",
      "label": "Social media text with emoji",
      "category": "social",
      "script": "mixed",
      "chars": 283,
      "words": 52,
      "o200k_base": 88,
      "cl100k_base": 95,
      "p50k_base": 96,
      "r50k_base": 96,
      "claude-opus-4-8": 144,
      "claude-sonnet-4-6": 114,
      "claude-haiku-4-5": 114,
      "tok_per_word_o200k": 1.692,
      "tok_per_100chars_o200k": 31.1,
      "tok_per_word_cl100k": 1.827,
      "tok_per_100chars_cl100k": 33.57,
      "tok_per_word_opus-4-8": 2.769,
      "tok_per_100chars_opus-4-8": 50.88
    },
    {
      "id": "S4-numeric",
      "label": "Numeric / tabular data",
      "category": "structured",
      "script": "latin",
      "chars": 416,
      "words": 10,
      "o200k_base": 237,
      "cl100k_base": 237,
      "p50k_base": 243,
      "r50k_base": 243,
      "claude-opus-4-8": 262,
      "claude-sonnet-4-6": 239,
      "claude-haiku-4-5": 239,
      "tok_per_word_o200k": 23.7,
      "tok_per_100chars_o200k": 56.97,
      "tok_per_word_cl100k": 23.7,
      "tok_per_100chars_cl100k": 56.97,
      "tok_per_word_opus-4-8": 26.2,
      "tok_per_100chars_opus-4-8": 62.98
    }
  ],
  "cost_per_volume": [
    {
      "id": "L1-en",
      "label": "English prose",
      "usd_per_1M_words_gpt-5": 1.46,
      "usd_per_1M_chars_gpt-5": 0.24,
      "usd_per_1M_words_gpt-5-mini": 0.29,
      "usd_per_1M_chars_gpt-5-mini": 0.05,
      "usd_per_1M_words_gpt-4o": 2.93,
      "usd_per_1M_chars_gpt-4o": 0.48,
      "usd_per_1M_words_claude-opus-4-8": 9.41,
      "usd_per_1M_chars_claude-opus-4-8": 1.55,
      "usd_per_1M_words_claude-sonnet-4-6": 3.7,
      "usd_per_1M_chars_claude-sonnet-4-6": 0.61,
      "usd_per_1M_words_claude-haiku-4-5": 1.23,
      "usd_per_1M_chars_claude-haiku-4-5": 0.2
    },
    {
      "id": "L2-es",
      "label": "Spanish prose",
      "usd_per_1M_words_gpt-5": 1.67,
      "usd_per_1M_chars_gpt-5": 0.28,
      "usd_per_1M_words_gpt-5-mini": 0.33,
      "usd_per_1M_chars_gpt-5-mini": 0.06,
      "usd_per_1M_words_gpt-4o": 3.34,
      "usd_per_1M_chars_gpt-4o": 0.55,
      "usd_per_1M_words_claude-opus-4-8": 11.96,
      "usd_per_1M_chars_claude-opus-4-8": 1.98,
      "usd_per_1M_words_claude-sonnet-4-6": 5.16,
      "usd_per_1M_chars_claude-sonnet-4-6": 0.85,
      "usd_per_1M_words_claude-haiku-4-5": 1.72,
      "usd_per_1M_chars_claude-haiku-4-5": 0.28
    },
    {
      "id": "L3-pt",
      "label": "Portuguese prose",
      "usd_per_1M_words_gpt-5": 1.68,
      "usd_per_1M_chars_gpt-5": 0.28,
      "usd_per_1M_words_gpt-5-mini": 0.34,
      "usd_per_1M_chars_gpt-5-mini": 0.06,
      "usd_per_1M_words_gpt-4o": 3.36,
      "usd_per_1M_chars_gpt-4o": 0.56,
      "usd_per_1M_words_claude-opus-4-8": 11.81,
      "usd_per_1M_chars_claude-opus-4-8": 1.97,
      "usd_per_1M_words_claude-sonnet-4-6": 5.53,
      "usd_per_1M_chars_claude-sonnet-4-6": 0.92,
      "usd_per_1M_words_claude-haiku-4-5": 1.84,
      "usd_per_1M_chars_claude-haiku-4-5": 0.31
    },
    {
      "id": "L4-fr",
      "label": "French prose",
      "usd_per_1M_words_gpt-5": 1.75,
      "usd_per_1M_chars_gpt-5": 0.28,
      "usd_per_1M_words_gpt-5-mini": 0.35,
      "usd_per_1M_chars_gpt-5-mini": 0.06,
      "usd_per_1M_words_gpt-4o": 3.51,
      "usd_per_1M_chars_gpt-4o": 0.56,
      "usd_per_1M_words_claude-opus-4-8": 12.61,
      "usd_per_1M_chars_claude-opus-4-8": 2.0,
      "usd_per_1M_words_claude-sonnet-4-6": 5.7,
      "usd_per_1M_chars_claude-sonnet-4-6": 0.91,
      "usd_per_1M_words_claude-haiku-4-5": 1.9,
      "usd_per_1M_chars_claude-haiku-4-5": 0.3
    },
    {
      "id": "L5-de",
      "label": "German prose",
      "usd_per_1M_words_gpt-5": 2.14,
      "usd_per_1M_chars_gpt-5": 0.3,
      "usd_per_1M_words_gpt-5-mini": 0.43,
      "usd_per_1M_chars_gpt-5-mini": 0.06,
      "usd_per_1M_words_gpt-4o": 4.27,
      "usd_per_1M_chars_gpt-4o": 0.6,
      "usd_per_1M_words_claude-opus-4-8": 17.42,
      "usd_per_1M_chars_claude-opus-4-8": 2.44,
      "usd_per_1M_words_claude-sonnet-4-6": 7.9,
      "usd_per_1M_chars_claude-sonnet-4-6": 1.11,
      "usd_per_1M_words_claude-haiku-4-5": 2.63,
      "usd_per_1M_chars_claude-haiku-4-5": 0.37
    },
    {
      "id": "L6-zh",
      "label": "Chinese prose (Simplified)",
      "usd_per_1M_chars_gpt-5": 1.04,
      "usd_per_1M_chars_gpt-5-mini": 0.21,
      "usd_per_1M_chars_gpt-4o": 2.08,
      "usd_per_1M_chars_claude-opus-4-8": 5.65,
      "usd_per_1M_chars_claude-sonnet-4-6": 3.41,
      "usd_per_1M_chars_claude-haiku-4-5": 1.14
    },
    {
      "id": "L7-ja",
      "label": "Japanese prose",
      "usd_per_1M_chars_gpt-5": 1.1,
      "usd_per_1M_chars_gpt-5-mini": 0.22,
      "usd_per_1M_chars_gpt-4o": 2.19,
      "usd_per_1M_chars_claude-opus-4-8": 5.13,
      "usd_per_1M_chars_claude-sonnet-4-6": 3.09,
      "usd_per_1M_chars_claude-haiku-4-5": 1.03
    }
  ]
}