js-tokens 9.0.1

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify pkg:npm/js-tokens@9.0.1
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/js-tokens@9.0.1
Constraints:
  - executionContext: node
Required runtime conditions:
  - ecosystem: npm
  - language: javascript
  - moduleSystem: cjs
  - packageManager: npm@10.9.8
  - runtime: node@22.23.2

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:f39a58783c94076348c759449624c8ac8f6dd46517146b97e6cba7dd1de2b0b3","constraints":{"executionContext":"node"},"contract":["jsTokens yields token objects with type and value for JavaScript code constructs","tokenize parses identifiers, keywords, punctuators, and numeric literals","tokenize handles regular expression literals and string literals with closed flag","tokenize supports JSX syntax when jsx option is enabled","tokenize handles template literals and template substitutions","extractSignificantTokens filters whitespace and comments"],"goal":"verify pkg:npm/js-tokens@9.0.1","kind":"HOW","packages":["pkg:npm/js-tokens@9.0.1"],"schemaVersion":1},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"cjs","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/js-tokens@9.0.1"],"schemaVersion":1,"subject":"pkg:npm/js-tokens@9.0.1","verifierAdapter":"node-typescript@1"}

{
  "name": "sample-js-tokens",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-js-tokens",
      "version": "1.0.0",
      "license": "MIT-0",
      "dependencies": {
        "js-tokens": "9.0.1"
      }
    },
    "node_modules/js-tokens": {
      "version": "9.0.1",
      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz",
      "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==",
      "license": "MIT"
    }
  }
}

{
  "name": "sample-js-tokens",
  "version": "1.0.0",
  "private": true,
  "description": "Clean-room verification of js-tokens",
  "main": "src/index.js",
  "license": "MIT-0",
  "dependencies": {
    "js-tokens": "9.0.1"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify pkg:npm/js-tokens@9.0.1",
  "kind": "HOW",
  "packages": [
    "pkg:npm/js-tokens@9.0.1"
  ],
  "constraints": {
    "executionContext": "node"
  },
  "runtimeConditions": {
    "ecosystem": "npm",
    "language": "javascript",
    "moduleSystem": "cjs",
    "packageManager": "npm@10.9.8",
    "runtime": "node@22.23.2"
  }
}

const jsTokens = require('js-tokens');

/**
 * Tokenize JavaScript code into an array of token objects.
 * @param {string} input - JavaScript code to tokenize.
 * @param {object} [options] - Tokenizer options.
 * @param {boolean} [options.jsx=false] - Whether to tokenize JSX syntax.
 * @returns {Array<{type: string, value: string, closed?: boolean}>}
 */
function tokenize(input, options = {}) {
  return Array.from(jsTokens(input, options));
}

/**
 * Extract non-whitespace and non-comment token values.
 * @param {string} input - JavaScript code.
 * @param {object} [options] - Tokenizer options.
 * @returns {string[]} Array of significant token string values.
 */
function extractSignificantTokens(input, options = {}) {
  const tokens = tokenize(input, options);
  return tokens
    .filter(
      (t) =>
        t.type !== 'WhiteSpace' &&
        t.type !== 'LineTerminatorSequence' &&
        t.type !== 'SingleLineComment' &&
        t.type !== 'MultiLineComment' &&
        t.type !== 'HashbangComment'
    )
    .map((t) => t.value);
}

module.exports = {
  jsTokens,
  tokenize,
  extractSignificantTokens
};

import assert from 'node:assert';
import { createRequire } from 'node:module';

const require = createRequire(import.meta.url);
const { jsTokens, tokenize, extractSignificantTokens } = require('../src/index.js');

// 1. Export verification
assert.strictEqual(typeof jsTokens, 'function', 'jsTokens should be exported as a function');
assert.strictEqual(typeof tokenize, 'function', 'tokenize should be exported as a function');
assert.strictEqual(typeof extractSignificantTokens, 'function', 'extractSignificantTokens should be exported as a function');

const generatorResult = jsTokens('let x = 1;');
const firstToken = generatorResult.next().value;
assert.strictEqual(firstToken.type, 'IdentifierName');
assert.strictEqual(firstToken.value, 'let');

// 2. Tokenize basic JavaScript declarations and statements
const basicTokens = tokenize('const answer = 42;');
assert.strictEqual(basicTokens.length, 8);
assert.deepStrictEqual(
  basicTokens.map((t) => t.type),
  [
    'IdentifierName',
    'WhiteSpace',
    'IdentifierName',
    'WhiteSpace',
    'Punctuator',
    'WhiteSpace',
    'NumericLiteral',
    'Punctuator'
  ]
);
assert.strictEqual(basicTokens[0].value, 'const');
assert.strictEqual(basicTokens[2].value, 'answer');
assert.strictEqual(basicTokens[4].value, '=');
assert.strictEqual(basicTokens[6].value, '42');
assert.strictEqual(basicTokens[7].value, ';');

// 3. Tokenize regular expression literals and string literals with closed status
const regexTokens = tokenize('/^[a-z]+$/i;');
assert.strictEqual(regexTokens[0].type, 'RegularExpressionLiteral');
assert.strictEqual(regexTokens[0].value, '/^[a-z]+$/i');
assert.strictEqual(regexTokens[0].closed, true);

const closedString = tokenize('"clean room"');
assert.strictEqual(closedString[0].type, 'StringLiteral');
assert.strictEqual(closedString[0].value, '"clean room"');
assert.strictEqual(closedString[0].closed, true);

const unclosedString = tokenize('"unclosed');
assert.strictEqual(unclosedString[0].type, 'StringLiteral');
assert.strictEqual(unclosedString[0].closed, false);

// 4. Tokenize JSX elements when jsx option is true
const jsxCode = '<Button onClick={handleClick}>Click Me</Button>';
const jsxTokens = tokenize(jsxCode, { jsx: true });
const tokenTypes = jsxTokens.map((t) => t.type);

assert.ok(tokenTypes.includes('JSXIdentifier'), 'Should identify JSX tag names');
assert.ok(tokenTypes.includes('JSXPunctuator'), 'Should identify JSX punctuators');
assert.ok(tokenTypes.includes('JSXText'), 'Should identify JSX text content');

const jsxIdentifiers = jsxTokens.filter((t) => t.type === 'JSXIdentifier').map((t) => t.value);
assert.deepStrictEqual(jsxIdentifiers, ['Button', 'onClick', 'Button']);

// 5. Tokenize template literals and substitutions
const templateTokens = tokenize('`User ${name} logged in.`');
assert.strictEqual(templateTokens[0].type, 'TemplateHead');
assert.strictEqual(templateTokens[0].value, '`User ${');
assert.strictEqual(templateTokens[1].type, 'IdentifierName');
assert.strictEqual(templateTokens[1].value, 'name');
assert.strictEqual(templateTokens[2].type, 'TemplateTail');
assert.strictEqual(templateTokens[2].value, '} logged in.`');
assert.strictEqual(templateTokens[2].closed, true);

// 6. Extract significant tokens excluding whitespace and comments
const sourceWithComments = '/* preamble */ let total = 10 + 20; // end of line';
const significant = extractSignificantTokens(sourceWithComments);
assert.deepStrictEqual(significant, ['let', 'total', '=', '10', '+', '20', ';']);

console.log('Contract tests passed successfully.');