"""Local public-text measurement; no model calls, API billing or hidden prompts."""
from pathlib import Path
import hashlib
import importlib.metadata
import json
import platform
import tiktoken

root = Path(__file__).resolve().parent
encoding = tiktoken.get_encoding('o200k_base')
document = (root / 'document.txt').read_text(encoding='utf-8')
instruction = 'Объясни руководителю порядок работы со сделками в Битрикс24 по этому документу. Отдели действия менеджера от настроек администратора. Не добавляй отсутствующие в источнике требования.'
history_unit = 'Пользователь: разбираем продажи для учебного примера. Ассистент: используем только публичную документацию, реальные данные компании не передаются.\n'
def measure(text):
    tokens = encoding.encode(text)
    assert encoding.decode(tokens) == text
    return len(tokens)

rows = []
for name, copies in [('short_history', 1), ('extended_synthetic_history', 120)]:
    history = history_unit * copies
    body = '\n\n'.join([instruction, history, document])
    (root / f'{name}.txt').write_text(body, encoding='utf-8')
    known_input = measure(body)
    rows.append({
        'scenario': name, 'history_copies': copies,
        'document_tokens': measure(document), 'instruction_tokens': measure(instruction),
        'history_tokens': measure(history), 'joined_text_tokens': known_input,
        'joined_text_sha256': hashlib.sha256(body.encode()).hexdigest(),
        'illustrative_capacity_tokens': 8192, 'illustrative_output_reserve_tokens': 2000,
        'arithmetic_remaining_tokens': 8192 - 2000 - known_input,
        'component_sum_difference': known_input - sum(map(measure, [instruction, history, document]))
    })
result = {
    'measurement': 'actual local tokenization of exact visible UTF-8 text',
    'python': platform.python_version(), 'tiktoken_version': importlib.metadata.version('tiktoken'),
    'encoding': encoding.name,
    'dependencies': {p: importlib.metadata.version(p) for p in ['tiktoken', 'regex', 'requests']},
    'document_sha256': hashlib.sha256((root / 'document.txt').read_bytes()).hexdigest(),
    'rows': rows,
    'limitations': [
        '8192 and 2000 are illustrative accounting assumptions, not limits of a named model or product.',
        'Counts include extracted Russian Bitrix24 article text with both desktop/mobile tabs; images and linked documents are not loaded.',
        'Visible joined text only: no API message framing, tool definitions, hidden system prompt, image tokens or reasoning measured.',
        'Extended history is synthetic repetition, not a real user conversation.',
        'Encoding-specific count; do not transfer these counts to Claude or Gemini.',
        'No generation, quality/recall benchmark, overflow API request, ERP implementation or customer case was performed.'
    ]
}
(root / 'measurement.v2.json').write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
print(json.dumps(result, ensure_ascii=False, indent=2))