charset-normalizer 3.4.9

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify pkg:pypi/charset-normalizer@3.4.9
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:pypi/charset-normalizer@3.4.9

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:eca398f805254da061a69c4051958ae4b9854314f8043fb96ec48889d87bfa27","contract":["charset_normalizer.from_bytes(data) detects encoding from raw byte sequences","matches.best() returns the highest-scoring CharsetMatch object","best.encoding provides the normalized encoding identifier","str(best) provides the decoded unicode string","best.byte_order_mark identifies the presence of UTF-8 BOM","from_bytes raises TypeError when input is not bytes or bytearray"],"goal":"verify pkg:pypi/charset-normalizer@3.4.9","kind":"HOW","packages":["pkg:pypi/charset-normalizer@3.4.9"],"schemaVersion":1},"contractCommand":["python","test/contract.py"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"pypi","libc":"glibc","libcVersion":"2.39","os":"linux","osVersionBucket":"24","packageManager":"pip","schemaVersion":1},"license":"MIT-0","packages":["pkg:pypi/charset-normalizer@3.4.9"],"schemaVersion":1,"subject":"pkg:pypi/charset-normalizer@3.4.9","verifierAdapter":"python@1"}

charset-normalizer==3.4.9

{
  "schemaVersion": 1,
  "goal": "verify pkg:pypi/charset-normalizer@3.4.9",
  "kind": "HOW",
  "packages": [
    "pkg:pypi/charset-normalizer@3.4.9"
  ]
}

"""Charset detector implementation using charset_normalizer."""

from typing import Any, Dict, Optional
import charset_normalizer
from charset_normalizer import CharsetMatch, CharsetMatches


def detect_encoding(data: bytes) -> Dict[str, Any]:
    """Detect character encoding and decode input byte sequence.

    Args:
        data: Raw byte string to detect encoding for.

    Returns:
        dict containing 'encoding', 'confidence', 'text', and 'chaos'.

    Raises:
        TypeError: If data is not bytes or bytearray.
    """
    if not isinstance(data, (bytes, bytearray)):
        raise TypeError(f"Expected bytes or bytearray, got {type(data).__name__}")

    matches: CharsetMatches = charset_normalizer.from_bytes(data)
    best: Optional[CharsetMatch] = matches.best()
    if best is None:
        return {
            "encoding": None,
            "confidence": 0.0,
            "text": "",
            "chaos": 1.0,
            "bom": False,
        }

    return {
        "encoding": best.encoding,
        "confidence": round(1.0 - best.chaos, 4),
        "text": str(best),
        "chaos": best.chaos,
        "bom": best.byte_order_mark,
    }


def detect_fast(data: bytes) -> Dict[str, Any]:
    """Detect character encoding using charset_normalizer.detect.

    Args:
        data: Raw byte string to detect encoding for.

    Returns:
        dict containing 'encoding', 'language', and 'confidence'.

    Raises:
        TypeError: If data is not bytes or bytearray.
    """
    if not isinstance(data, (bytes, bytearray)):
        raise TypeError(f"Expected bytes or bytearray, got {type(data).__name__}")

    return charset_normalizer.detect(data)

"""Contract verification test for pkg:pypi/charset-normalizer@3.4.9."""

import os
import sys

# Ensure repository root is on sys.path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import charset_normalizer
from charset_normalizer import CharsetMatch, CharsetMatches
from src.detector import detect_encoding, detect_fast


def test_from_bytes_utf8():
    text = "Hello, world! 🌍 Bonjour le monde! Café & thé."
    raw = text.encode("utf-8")

    matches = charset_normalizer.from_bytes(raw)
    assert isinstance(matches, CharsetMatches), f"Expected CharsetMatches, got {type(matches)}"
    assert len(matches) > 0, "Expected at least one match"

    best = matches.best()
    assert isinstance(best, CharsetMatch), f"Expected CharsetMatch, got {type(best)}"
    assert best.encoding == "utf_8", f"Expected utf_8, got {best.encoding}"
    assert str(best) == text, f"Decoded string mismatch: {str(best)}"
    assert best.raw == raw, "Raw bytes mismatch"
    assert best.chaos == 0.0, f"Expected 0.0 chaos, got {best.chaos}"
    assert best.byte_order_mark is False, "Expected no BOM"


def test_from_bytes_with_bom():
    text = "UTF-8 with BOM content"
    raw = b"\xef\xbb\xbf" + text.encode("utf-8")

    matches = charset_normalizer.from_bytes(raw)
    best = matches.best()
    assert best is not None, "Expected match for UTF-8 with BOM"
    assert best.encoding == "utf_8", f"Expected utf_8, got {best.encoding}"
    assert best.byte_order_mark is True, "Expected BOM to be True"
    assert best.bom is True, "Expected bom property to be True"
    assert str(best) == text, f"Decoded string mismatch: {str(best)}"


def test_from_bytes_ascii():
    text = "Pure ASCII text message for validation."
    raw = text.encode("ascii")

    matches = charset_normalizer.from_bytes(raw)
    assert len(matches) > 0, "Expected at least one match"
    best = matches.best()
    assert best is not None, "Expected match"
    assert best.encoding == "ascii", f"Expected ascii, got {best.encoding}"
    assert str(best) == text, f"Decoded string mismatch: {str(best)}"


def test_from_bytes_cp_isolation():
    text = "Привет мир!"
    raw = text.encode("cp1251")

    matches = charset_normalizer.from_bytes(raw, cp_isolation=["cp1251", "utf_8"])
    best = matches.best()
    assert best is not None, "Expected match with cp_isolation"
    assert best.encoding == "cp1251", f"Expected cp1251, got {best.encoding}"
    assert str(best) == text, f"Decoded string mismatch: {str(best)}"


def test_from_bytes_type_error():
    try:
        charset_normalizer.from_bytes(12345)
        assert False, "Expected TypeError when passing int to from_bytes"
    except TypeError:
        pass

    try:
        charset_normalizer.from_bytes("unicode string")
        assert False, "Expected TypeError when passing str to from_bytes"
    except TypeError:
        pass


def test_detector_helpers():
    sample_text = "Testing detector helper functions with non-ASCII characters: café, résumé & 🚀."
    raw_utf8 = sample_text.encode("utf-8")

    res = detect_encoding(raw_utf8)
    assert res["encoding"] == "utf_8", f"Expected utf_8, got {res['encoding']}"
    assert res["text"] == sample_text, f"Expected text match, got {res['text']}"
    assert res["confidence"] == 1.0, f"Expected 1.0 confidence, got {res['confidence']}"
    assert res["bom"] is False, "Expected bom False"

    fast_res = detect_fast(raw_utf8)
    assert isinstance(fast_res, dict), "Expected dict from detect_fast"
    assert fast_res.get("encoding") == "utf-8", f"Expected utf-8, got {fast_res.get('encoding')}"
    assert fast_res.get("confidence") == 1.0, f"Expected 1.0 confidence, got {fast_res.get('confidence')}"


def main():
    test_from_bytes_utf8()
    test_from_bytes_with_bom()
    test_from_bytes_ascii()
    test_from_bytes_cp_isolation()
    test_from_bytes_type_error()
    test_detector_helpers()
    print("All contract assertions passed successfully.")
    return 0


if __name__ == "__main__":
    sys.exit(main())