Token 计数�?- WebUtils

输入文本

<button class="btn btn-primary" onclick="countTokens()">📊 计算 Token</button>
            <button class="btn btn-secondary" onclick="clearInput()">🗑�?清空</button>
            <button class="btn btn-secondary" onclick="loadSample()">📝 加载示例</button>
            <button class="btn btn-secondary" onclick="pasteFromClipboard()">📋 粘贴</button>
          </div>

<div class="token-visualization" id="tokenVisualization" style="display: none">
            <div class="card-title">Token 可视�?/div>
            <div id="tokenDisplay"></div>
          </div>
        </div>

<div class="sidebar">
          <div class="card model-selector">
            <div class="card-title">选择模型</div>
            <select id="modelSelect" onchange="countTokens()">
              <optgroup label="OpenAI">
                <option value="gpt-4o">GPT-4o / GPT-4o mini</option>
                <option value="gpt-4-turbo">GPT-4 Turbo</option>
                <option value="gpt-4">GPT-4</option>
                <option value="gpt-3.5-turbo">GPT-3.5 Turbo</option>
              </optgroup>
              <optgroup label="Anthropic">
                <option value="claude-3.5-sonnet">Claude 3.5 Sonnet</option>
                <option value="claude-3-opus">Claude 3 Opus</option>
                <option value="claude-3-haiku">Claude 3 Haiku</option>
              </optgroup>
              <optgroup label="DeepSeek">
                <option value="deepseek-v3">DeepSeek V3</option>
                <option value="deepseek-r1">DeepSeek R1</option>
              </optgroup>
              <optgroup label="Google">
                <option value="gemini-2.0">Gemini 2.0 Flash</option>
                <option value="gemini-1.5-pro">Gemini 1.5 Pro</option>
              </optgroup>
            </select>
          </div>

<div class="card">
            <div class="card-title">统计结果</div>
            <div class="stats-grid">
              <div class="stat-item full-width">
                <div class="stat-value" id="tokenCount">0</div>
                <div class="stat-label">Token 数量</div>
              </div>
              <div class="stat-item">
                <div class="stat-value" id="charCount">0</div>
                <div class="stat-label">字符�?/div>
              </div>
              <div class="stat-item">
                <div class="stat-value" id="wordCount">0</div>
                <div class="stat-label">单词/词数</div>
              </div>
              <div class="stat-item">
                <div class="stat-value" id="lineCount">0</div>
                <div class="stat-label">行数</div>
              </div>
              <div class="stat-item">
                <div class="stat-value" id="ratio">0</div>
                <div class="stat-label">字符/Token �?/div>
              </div>
            </div>
          </div>

<div class="card">
            <div class="card-title">💰 费用估算</div>
            <div class="cost-breakdown">
              <div class="cost-item">
                <span class="cost-label">输入费用</span>
                <span class="cost-value" id="inputCost">$0.0000</span>
              </div>
              <div class="cost-item">
                <span class="cost-label">输出费用 (估算)</span>
                <span class="cost-value" id="outputCost">$0.0000</span>
              </div>
              <div class="cost-item">
                <span class="cost-label">总计</span>
                <span class="cost-value" id="totalCost">$0.0000</span>
              </div>
            </div>
            <div class="note">* 输出费用按输�?Token �?50% 估算</div>
          </div>
        </div>
      </div>

<div class="info-section">
        <div class="info-card">
          <h3>📚 模型定价参�?(每百�?Token)</h3>
          <table class="info-table">
            <thead>
              <tr>
                <th>模型</th>
                <th>输入价格</th>
                <th>输出价格</th>
                <th>上下文窗�?/th>
              </tr>
            </thead>
            <tbody>
              <tr>
                <td>GPT-4o</td>
                <td>$2.50</td>
                <td>$10.00</td>
                <td>128K</td>
              </tr>
              <tr>
                <td>GPT-4o mini</td>
                <td>$0.15</td>
                <td>$0.60</td>
                <td>128K</td>
              </tr>
              <tr>
                <td>Claude 3.5 Sonnet</td>
                <td>$3.00</td>
                <td>$15.00</td>
                <td>200K</td>
              </tr>
              <tr>
                <td>Claude 3 Opus</td>
                <td>$15.00</td>
                <td>$75.00</td>
                <td>200K</td>
              </tr>
              <tr>
                <td>DeepSeek V3</td>
                <td>$0.27</td>
                <td>$1.10</td>
                <td>64K</td>
              </tr>
              <tr>
                <td>DeepSeek R1</td>
                <td>$0.55</td>
                <td>$2.19</td>
                <td>64K</td>
              </tr>
              <tr>
                <td>Gemini 2.0 Flash</td>
                <td>$0.10</td>
                <td>$0.40</td>
                <td>1M</td>
              </tr>
            </tbody>
          </table>
        </div>
      </div>
    </div>

// Token estimation ratios (characters per token)
      const TOKEN_RATIOS = {
        "gpt-4o": { en: 4.0, zh: 1.5 },
        "gpt-4-turbo": { en: 4.0, zh: 1.5 },
        "gpt-4": { en: 4.0, zh: 1.5 },
        "gpt-3.5-turbo": { en: 4.0, zh: 1.5 },
        "claude-3.5-sonnet": { en: 4.0, zh: 1.8 },
        "claude-3-opus": { en: 4.0, zh: 1.8 },
        "claude-3-haiku": { en: 4.0, zh: 1.8 },
        "deepseek-v3": { en: 4.0, zh: 2.0 },
        "deepseek-r1": { en: 4.0, zh: 2.0 },
        "gemini-2.0": { en: 4.0, zh: 1.8 },
        "gemini-1.5-pro": { en: 4.0, zh: 1.8 }
      };

// Theme handling
      function initTheme() {
        const savedTheme = localStorage.getItem("theme") || "dark";
        document.documentElement.setAttribute("data-theme", savedTheme);
      }

window.toggleTheme = function () {
        const current = document.documentElement.getAttribute("data-theme");
        const newTheme = current === "light" ? "dark" : "light";
        document.documentElement.setAttribute("data-theme", newTheme);
        localStorage.setItem("theme", newTheme);
      };

// Estimate tokens based on character analysis
      function estimateTokens(text, model) {
        if (!text) return 0;

const ratios = TOKEN_RATIOS[model] || TOKEN_RATIOS["gpt-4o"];

// Count Chinese and non-Chinese characters
        const chineseChars = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || []).length;
        const otherChars = text.length - chineseChars;

// Estimate tokens
        const chineseTokens = chineseChars / ratios.zh;
        const otherTokens = otherChars / ratios.en;

return Math.ceil(chineseTokens + otherTokens);
      }

// Tokenize text for visualization (simplified)
      function tokenizeForVisualization(text) {
        const tokens = [];
        let current = "";

for (let i = 0; i < text.length; i++) {
          const char = text[i];

// Chinese character - each is roughly a token
          if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(char)) {
            if (current) {
              tokens.push(current);
              current = "";
            }
            tokens.push(char);
          }
          // Whitespace
          else if (/\s/.test(char)) {
            if (current) {
              tokens.push(current);
              current = "";
            }
            if (char === "\n") {
              tokens.push("�?);
            }
          }
          // Punctuation
          else if (/[^\w]/.test(char)) {
            if (current) {
              tokens.push(current);
              current = "";
            }
            tokens.push(char);
          }
          // Regular characters
          else {
            current += char;
            // Split long words
            if (current.length >= 4) {
              tokens.push(current);
              current = "";
            }
          }
        }

if (current) {
          tokens.push(current);
        }

return tokens;
      }

// Count tokens and update UI
      function countTokens() {
        const text = document.getElementById("inputText").value;
        const model = document.getElementById("modelSelect").value;

// Basic stats
        const charCount = text.length;
        const wordCount = text.trim() ? text.trim().split(/\s+/).length : 0;
        const lineCount = text ? text.split("\n").length : 0;

// Token estimation
        const tokenCount = estimateTokens(text, model);
        const ratio = tokenCount > 0 ? (charCount / tokenCount).toFixed(1) : 0;

// Update stats display
        document.getElementById("tokenCount").textContent = tokenCount.toLocaleString();
        document.getElementById("charCount").textContent = charCount.toLocaleString();
        document.getElementById("wordCount").textContent = wordCount.toLocaleString();
        document.getElementById("lineCount").textContent = lineCount.toLocaleString();
        document.getElementById("ratio").textContent = ratio;

// Calculate costs
        const pricing = MODEL_PRICING[model];
        const inputCost = (tokenCount / 1000000) * pricing.input;
        const outputCost = ((tokenCount * 0.5) / 1000000) * pricing.output;
        const totalCost = inputCost + outputCost;

document.getElementById("inputCost").textContent = "$" + inputCost.toFixed(6);
        document.getElementById("outputCost").textContent = "$" + outputCost.toFixed(6);
        document.getElementById("totalCost").textContent = "$" + totalCost.toFixed(6);

// Token visualization
        if (text && text.length < 2000) {
          const tokens = tokenizeForVisualization(text);
          const display = document.getElementById("tokenDisplay");
          display.innerHTML = tokens
            .map(
              (t) => `<span class="token">${t.replace(/</g, "<").replace(/>/g, ">")}</span>`
            )
            .join("");
          document.getElementById("tokenVisualization").style.display = "block";
        } else {
          document.getElementById("tokenVisualization").style.display = "none";
        }
      }

window.clearInput = function () {
        document.getElementById("inputText").value = "";
        countTokens();
      };

window.loadSample = function () {
        const samples = [
          `你好，我是一�?AI 助手。今天我来帮你了�?Token 是什么�?

Token 是大语言模型处理文本的基本单位。对于英文，一�?Token 大约�?4 个字符或 3/4 个单词。对于中文，一个汉字通常对应 1-2 �?Token�?

Here's an example in English: "Hello, world!" might be tokenized as ["Hello", ",", " world", "!"].

费用计算公式: 总费�?= (输入Token × 输入单价) + (输出Token × 输出单价)`,

`const calculateTokenCost = (tokens, model) => {
    const pricing = {
        'gpt-4o': { input: 2.50, output: 10.00 },
        'claude-3.5-sonnet': { input: 3.00, output: 15.00 },
        'deepseek-v3': { input: 0.27, output: 1.10 }
    };
    
    const price = pricing[model];
    return {
        inputCost: (tokens.input / 1000000) * price.input,
        outputCost: (tokens.output / 1000000) * price.output
    };
};`,

`# API 请求示例

## Python
\`\`\`python
import openai

response = openai.ChatCompletion.create(
    model="gpt-4o",
    messages=[
        {"role": "system", "content": "你是一个有帮助的助�?},
        {"role": "user", "content": "解释什么是 Token"}
    ]
)
\`\`\`

## 费用估算
- 输入: ~50 tokens × $2.50/1M = $0.000125
- 输出: ~200 tokens × $10.00/1M = $0.002
- 总计: ~$0.002125`
        ];

document.getElementById("inputText").value =
          samples[Math.floor(Math.random() * samples.length)];
        countTokens();
      };

// Auto-count on input
      document.getElementById("inputText").addEventListener("input", countTokens);

// Initialize
      initTheme();
      countTokens();
    </script>
  </body>
</html>