@csstools/css-tokenizer 4.0.0: tokenize CSS string into tokens with @csstools/css-tokenizer

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: tokenize CSS string into tokens with @csstools/css-tokenizer
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/%40csstools/css-tokenizer@4.0.0
Demonstrate these symbols/APIs:
  - tokenize

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:a1757b70ebf19c36608b47fd8a858dc97956e6494bbbc4537bc781292fbcf9f0","contract":["tokenize is exported as a function from @csstools/css-tokenizer","tokenize decomposes standard CSS declarations into CSSToken tuples containing type, raw representation, start and end offsets","tokenize extracts structured token values for identifiers and dimensions with parsed units and numeric types","tokenize with onParseError callback captures parser errors without throwing an unhandled exception","stringify reconstructs valid CSS text from an array of tokens produced by tokenize"],"goal":"tokenize CSS string into tokens with @csstools/css-tokenizer","kind":"HOW","packages":["pkg:npm/%40csstools/css-tokenizer@4.0.0"],"schemaVersion":1,"symbols":["tokenize"]},"contractCommand":["node","test/contract.js"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"esm","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/%40csstools/css-tokenizer@4.0.0"],"schemaVersion":1,"subject":"pkg:npm/%40csstools/css-tokenizer@4.0.0","symbols":["tokenize"],"verifierAdapter":"node-typescript@1"}

{
  "name": "sample-csstools-css-tokenizer",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-csstools-css-tokenizer",
      "version": "1.0.0",
      "license": "MIT-0",
      "dependencies": {
        "@csstools/css-tokenizer": "4.0.0"
      }
    },
    "node_modules/@csstools/css-tokenizer": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz",
      "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==",
      "license": "MIT",
      "engines": {
        "node": ">=20.19.0"
      }
    }
  }
}

{
  "name": "sample-csstools-css-tokenizer",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "description": "Clean-room verification for @csstools/css-tokenizer tokenize",
  "main": "src/index.js",
  "scripts": {
    "test": "node test/contract.js"
  },
  "dependencies": {
    "@csstools/css-tokenizer": "4.0.0"
  },
  "license": "MIT-0"
}

{
  "schemaVersion": 1,
  "goal": "tokenize CSS string into tokens with @csstools/css-tokenizer",
  "kind": "HOW",
  "packages": [
    "pkg:npm/%40csstools/css-tokenizer@4.0.0"
  ],
  "symbols": [
    "tokenize"
  ]
}

import {
  tokenize,
  tokenizer,
  TokenType,
  isToken,
  isTokenIdent,
  isTokenDimension,
  stringify,
  ParseError
} from '@csstools/css-tokenizer';

/**
 * Tokenizes a CSS string into an array of CSSToken items.
 * @param {string} css
 * @param {object} [options]
 * @returns {Array}
 */
export function tokenizeCSS(css, options = {}) {
  return tokenize({ css, unicodeRangesAllowed: options.unicodeRangesAllowed }, options);
}

/**
 * Filters an array of tokens by token type.
 * @param {Array} tokens
 * @param {string} type
 * @returns {Array}
 */
export function filterTokensByType(tokens, type) {
  return tokens.filter((token) => isToken(token) && token[0] === type);
}

/**
 * Tokenizes CSS while collecting any parsing errors encountered.
 * @param {string} css
 * @returns {{ tokens: Array, errors: Array<ParseError> }}
 */
export function tokenizeWithErrors(css) {
  const errors = [];
  const tokens = tokenize(
    { css },
    {
      onParseError: (err) => {
        errors.push(err);
      }
    }
  );
  return { tokens, errors };
}

/**
 * Reconstructs CSS string representation from an array of tokens.
 * @param {Array} tokens
 * @returns {string}
 */
export function stringifyTokens(tokens) {
  return stringify(...tokens);
}

export {
  tokenize,
  tokenizer,
  TokenType,
  isToken,
  isTokenIdent,
  isTokenDimension,
  stringify,
  ParseError
};

import assert from 'node:assert/strict';
import {
  tokenize,
  TokenType,
  isToken,
  isTokenIdent,
  isTokenDimension,
  stringify,
  tokenizeCSS,
  filterTokensByType,
  tokenizeWithErrors,
  stringifyTokens
} from '../src/index.js';

// 1. tokenize is exported as a function from @csstools/css-tokenizer
assert.equal(typeof tokenize, 'function', 'tokenize must be a function');
assert.equal(typeof tokenizeCSS, 'function', 'tokenizeCSS helper must be a function');

// 2. tokenize decomposes standard CSS declarations into CSSToken tuples containing type, raw representation, start and end offsets
const cssSample = 'color: red; margin: 10px;';
const tokens = tokenizeCSS(cssSample);

assert.ok(Array.isArray(tokens), 'tokens must be an array');
assert.ok(tokens.length > 0, 'tokens array must not be empty');

for (const token of tokens) {
  assert.ok(isToken(token), 'each element must satisfy isToken');
  assert.equal(typeof token[0], 'string', 'token[0] (type) must be string');
  assert.equal(typeof token[1], 'string', 'token[1] (raw) must be string');
  assert.equal(typeof token[2], 'number', 'token[2] (start) must be number');
  assert.equal(typeof token[3], 'number', 'token[3] (end) must be number');
}

const firstToken = tokens[0];
assert.equal(firstToken[0], TokenType.Ident);
assert.equal(firstToken[1], 'color');
assert.equal(firstToken[2], 0);
assert.equal(firstToken[3], 4);
assert.deepEqual(firstToken[4], { value: 'color' });
assert.ok(isTokenIdent(firstToken), 'first token must satisfy isTokenIdent');

const lastToken = tokens[tokens.length - 1];
assert.equal(lastToken[0], TokenType.EOF);

// 3. tokenize extracts structured token values for identifiers and dimensions with parsed units and numeric types
const dimensionTokens = filterTokensByType(tokens, TokenType.Dimension);
assert.equal(dimensionTokens.length, 1, 'must find exactly one dimension token');

const dimToken = dimensionTokens[0];
assert.ok(isTokenDimension(dimToken), 'must satisfy isTokenDimension');
assert.equal(dimToken[1], '10px');
assert.equal(dimToken[4].value, 10);
assert.equal(dimToken[4].unit, 'px');
assert.equal(dimToken[4].type, 'integer');

const identTokens = filterTokensByType(tokens, TokenType.Ident);
const identValues = identTokens.map((t) => t[4].value);
assert.deepEqual(identValues, ['color', 'red', 'margin']);

// 4. tokenize with onParseError callback captures parser errors without throwing an unhandled exception
const malformedCSS = '"unclosed string\n';
const { tokens: errTokens, errors } = tokenizeWithErrors(malformedCSS);

assert.ok(Array.isArray(errTokens), 'errTokens must be an array');
assert.ok(Array.isArray(errors), 'errors must be an array');
assert.ok(errors.length > 0, 'must report at least one parse error for unclosed newline string');
assert.ok(errors[0].message.includes('string'), 'error message must indicate string parse error');
assert.equal(typeof errors[0].sourceStart, 'number', 'error must contain sourceStart offset');
assert.equal(typeof errors[0].sourceEnd, 'number', 'error must contain sourceEnd offset');

// 5. stringify reconstructs valid CSS text from an array of tokens produced by tokenize
const reconstructedCSS = stringifyTokens(tokens);
assert.equal(reconstructedCSS, cssSample, 'stringifyTokens must reconstruct original CSS string');

const directStringify = stringify(...tokens);
assert.equal(directStringify, cssSample, 'direct stringify must match original CSS string');

console.log('All contract assertions passed successfully.');