js-tokens 10.0.0: default

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify js-tokens.default in pkg:npm/js-tokens@10.0.0
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/js-tokens@10.0.0
Demonstrate these symbols/APIs:
  - js-tokens.default

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:452e898f96cd618c2b96412f82085e4a4318e545b3c8137d4f0d670a08fa1ef8","contract":["js-tokens default export is a generator function","js-tokens default export tokenizes JavaScript statements into typed tokens","js-tokens default export indicates closed status on string and regex tokens","js-tokens default export parses JSX elements when jsx option is true","js-tokens default export parses template literals with substitutions","js-tokens default export yields Invalid token type for unrecognized characters"],"goal":"verify js-tokens.default in pkg:npm/js-tokens@10.0.0","kind":"HOW","packages":["pkg:npm/js-tokens@10.0.0"],"schemaVersion":1,"symbols":["js-tokens.default"]},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"esm","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/js-tokens@10.0.0"],"schemaVersion":1,"subject":"pkg:npm/js-tokens@10.0.0","symbols":["js-tokens.default"],"verifierAdapter":"node-typescript@1"}

{
  "name": "sample-js-tokens-default",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-js-tokens-default",
      "version": "1.0.0",
      "dependencies": {
        "js-tokens": "10.0.0"
      }
    },
    "node_modules/js-tokens": {
      "version": "10.0.0",
      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz",
      "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==",
      "license": "MIT"
    }
  }
}

{
  "name": "sample-js-tokens-default",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "description": "Clean-room verification of js-tokens.default",
  "main": "src/index.js",
  "dependencies": {
    "js-tokens": "10.0.0"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify js-tokens.default in pkg:npm/js-tokens@10.0.0",
  "kind": "HOW",
  "packages": [
    "pkg:npm/js-tokens@10.0.0"
  ],
  "symbols": [
    "js-tokens.default"
  ]
}

import jsTokens from 'js-tokens';

/**
 * Tokenizes JavaScript code into an array of token objects.
 * @param {string} input - The source code to tokenize.
 * @param {object} [options] - Options for tokenization.
 * @param {boolean} [options.jsx=false] - Whether JSX is supported.
 * @returns {Array<{type: string, value: string, closed?: boolean}>}
 */
export function tokenize(input, options = {}) {
  return Array.from(jsTokens(input, options));
}

/**
 * Collects significant tokens by filtering out whitespace and comments.
 * @param {string} input - The source code to tokenize.
 * @param {object} [options] - Options for tokenization.
 * @returns {string[]}
 */
export function getSignificantTokenValues(input, options = {}) {
  const ignoredTypes = new Set([
    'WhiteSpace',
    'LineTerminatorSequence',
    'SingleLineComment',
    'MultiLineComment',
    'HashbangComment'
  ]);
  return tokenize(input, options)
    .filter((token) => !ignoredTypes.has(token.type))
    .map((token) => token.value);
}

export { jsTokens };
export default jsTokens;

import assert from 'node:assert';
import jsTokens, { default as defaultExport } from 'js-tokens';
import { tokenize, getSignificantTokenValues } from '../src/index.js';

// 1. js-tokens default export is a generator function
assert.strictEqual(typeof jsTokens, 'function', 'default import must be a function');
assert.strictEqual(defaultExport, jsTokens, 'named default export must equal default import');
const sampleGen = jsTokens('42');
assert.strictEqual(typeof sampleGen[Symbol.iterator], 'function', 'default export must return an iterable');
assert.strictEqual(typeof sampleGen.next, 'function', 'default export must return an iterator');

// 2. js-tokens default export tokenizes JavaScript statements into typed tokens
const tokens = tokenize('const answer = 42;');
assert.strictEqual(tokens.length, 8);
assert.deepStrictEqual(
  tokens.map((t) => t.type),
  [
    'IdentifierName',
    'WhiteSpace',
    'IdentifierName',
    'WhiteSpace',
    'Punctuator',
    'WhiteSpace',
    'NumericLiteral',
    'Punctuator'
  ]
);
assert.strictEqual(tokens[0].value, 'const');
assert.strictEqual(tokens[2].value, 'answer');
assert.strictEqual(tokens[4].value, '=');
assert.strictEqual(tokens[6].value, '42');
assert.strictEqual(tokens[7].value, ';');

const significant = getSignificantTokenValues('/* header */ const answer = 42; // end');
assert.deepStrictEqual(significant, ['const', 'answer', '=', '42', ';']);

// 3. js-tokens default export indicates closed status on string and regex tokens
const closedString = tokenize('"verified"');
assert.strictEqual(closedString[0].type, 'StringLiteral');
assert.strictEqual(closedString[0].value, '"verified"');
assert.strictEqual(closedString[0].closed, true);

const unclosedString = tokenize('"unterminated');
assert.strictEqual(unclosedString[0].type, 'StringLiteral');
assert.strictEqual(unclosedString[0].value, '"unterminated');
assert.strictEqual(unclosedString[0].closed, false);

const closedRegex = tokenize('/^token[0-9]+$/g;');
assert.strictEqual(closedRegex[0].type, 'RegularExpressionLiteral');
assert.strictEqual(closedRegex[0].value, '/^token[0-9]+$/g');
assert.strictEqual(closedRegex[0].closed, true);

// 4. js-tokens default export parses JSX elements when jsx option is true
const jsxSource = '<Widget active={true}>Content</Widget>';
const jsxTokens = tokenize(jsxSource, { jsx: true });
const jsxTypes = jsxTokens.map((t) => t.type);

assert.ok(jsxTypes.includes('JSXIdentifier'), 'Should identify JSX identifiers');
assert.ok(jsxTypes.includes('JSXPunctuator'), 'Should identify JSX punctuators');
assert.ok(jsxTypes.includes('JSXText'), 'Should identify JSX text content');

const nonJsxTokens = tokenize('<Widget>', { jsx: false });
assert.strictEqual(nonJsxTokens[0].type, 'Punctuator');
assert.strictEqual(nonJsxTokens[0].value, '<');

// 5. js-tokens default export parses template literals with substitutions
const templateSource = '`User ${name}: ${role}`';
const templateTokens = tokenize(templateSource);
assert.strictEqual(templateTokens[0].type, 'TemplateHead');
assert.strictEqual(templateTokens[0].value, '`User ${');
assert.strictEqual(templateTokens[1].type, 'IdentifierName');
assert.strictEqual(templateTokens[1].value, 'name');
assert.strictEqual(templateTokens[2].type, 'TemplateMiddle');
assert.strictEqual(templateTokens[2].value, '}: ${');
assert.strictEqual(templateTokens[3].type, 'IdentifierName');
assert.strictEqual(templateTokens[3].value, 'role');
assert.strictEqual(templateTokens[4].type, 'TemplateTail');
assert.strictEqual(templateTokens[4].value, '}`');
assert.strictEqual(templateTokens[4].closed, true);

// 6. js-tokens default export yields Invalid token type for unrecognized characters
const invalidTokens = tokenize('\0');
assert.strictEqual(invalidTokens[0].type, 'Invalid');
assert.strictEqual(invalidTokens[0].value, '\0');

console.log('All js-tokens.default contract assertions passed.');