js-tokens 4.0.0

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify pkg:npm/js-tokens@4.0.0
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/js-tokens@4.0.0
Constraints:
  - executionContext: node
Required runtime conditions:
  - ecosystem: npm
  - language: javascript
  - moduleSystem: cjs
  - packageManager: npm@10.9.8
  - runtime: node@22.23.2

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:6124099d9c4429576d12341dc9868f92315ac08d1718617cfce6b8d10bf9e1a6","constraints":{"executionContext":"node"},"contract":["jsTokens.default is a global RegExp that matches JavaScript tokens","matchToToken converts RegExp match results into token objects with type, value, and closed properties","tokenize correctly classifies names, keywords, punctuators, numbers, and whitespace","tokenize handles single-line comments, block comments, and identifies unclosed comments","tokenize parses single-quoted, double-quoted, and template strings, marking closed or unclosed status","tokenize parses regular expression literals with flags"],"goal":"verify pkg:npm/js-tokens@4.0.0","kind":"HOW","packages":["pkg:npm/js-tokens@4.0.0"],"schemaVersion":1},"contractCommand":["node","test/contract.js"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"cjs","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/js-tokens@4.0.0"],"schemaVersion":1,"subject":"pkg:npm/js-tokens@4.0.0","verifierAdapter":"node-typescript@1"}

{
  "name": "sample-js-tokens",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-js-tokens",
      "version": "1.0.0",
      "license": "MIT-0",
      "dependencies": {
        "js-tokens": "4.0.0"
      }
    },
    "node_modules/js-tokens": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
      "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
      "license": "MIT"
    }
  }
}

{
  "name": "sample-js-tokens",
  "version": "1.0.0",
  "private": true,
  "description": "Clean-room verification sample for js-tokens",
  "main": "src/index.js",
  "scripts": {
    "test": "node test/contract.js"
  },
  "dependencies": {
    "js-tokens": "4.0.0"
  },
  "license": "MIT-0"
}

{
  "schemaVersion": 1,
  "goal": "verify pkg:npm/js-tokens@4.0.0",
  "kind": "HOW",
  "packages": [
    "pkg:npm/js-tokens@4.0.0"
  ],
  "constraints": {
    "executionContext": "node"
  },
  "runtimeConditions": {
    "ecosystem": "npm",
    "language": "javascript",
    "moduleSystem": "cjs",
    "packageManager": "npm@10.9.8",
    "runtime": "node@22.23.2"
  }
}

const jsTokens = require('js-tokens');

/**
 * Tokenizes a string of JavaScript code using js-tokens.
 * @param {string} code - The source code to tokenize.
 * @returns {Array<{type: string, value: string, closed: boolean|undefined}>} Array of tokens.
 */
function tokenize(code) {
  const tokens = [];
  const regex = new RegExp(jsTokens.default.source, jsTokens.default.flags);
  let match;
  while ((match = regex.exec(code)) !== null) {
    tokens.push(jsTokens.matchToToken(match));
  }
  return tokens;
}

/**
 * Filters tokens by a specific token type.
 * @param {string} code - The source code.
 * @param {string} type - The token type to filter by.
 * @returns {Array<{type: string, value: string, closed: boolean|undefined}>}
 */
function getTokensByType(code, type) {
  return tokenize(code).filter((token) => token.type === type);
}

module.exports = {
  jsTokens,
  tokenize,
  getTokensByType
};

const assert = require('node:assert');
const jsTokens = require('js-tokens');
const { tokenize, getTokensByType } = require('../src/index.js');

// 1. jsTokens exports default RegExp and matchToToken function
assert.ok(jsTokens.default instanceof RegExp, 'jsTokens.default must be a RegExp instance');
assert.strictEqual(typeof jsTokens.matchToToken, 'function', 'jsTokens.matchToToken must be a function');
assert.strictEqual(jsTokens.default.global, true, 'jsTokens.default must have the global flag enabled');

// 2. matchToToken converts match results into token objects
const sampleMatch = jsTokens.default.exec('var x = 10;');
assert.ok(Array.isArray(sampleMatch), 'exec result must be an array');
const sampleToken = jsTokens.matchToToken(sampleMatch);
assert.strictEqual(typeof sampleToken, 'object', 'token must be an object');
assert.strictEqual(sampleToken.type, 'name', 'first token must be type name');
assert.strictEqual(sampleToken.value, 'var', 'first token value must be "var"');

// 3. Tokenize classifies names, numbers, punctuators, and whitespace
const simpleCode = 'const count = 42;';
const simpleTokens = tokenize(simpleCode);
assert.deepStrictEqual(
  simpleTokens,
  [
    { type: 'name', value: 'const', closed: undefined },
    { type: 'whitespace', value: ' ', closed: undefined },
    { type: 'name', value: 'count', closed: undefined },
    { type: 'whitespace', value: ' ', closed: undefined },
    { type: 'punctuator', value: '=', closed: undefined },
    { type: 'whitespace', value: ' ', closed: undefined },
    { type: 'number', value: '42', closed: undefined },
    { type: 'punctuator', value: ';', closed: undefined }
  ],
  'tokens for "const count = 42;" must match expected structure'
);

// Number format variations
const numberCode = '0xFF 0b1010 0o777 3.14 1e10';
const numberTokens = getTokensByType(numberCode, 'number');
assert.strictEqual(numberTokens.length, 5, 'must parse all numeric literal formats');
assert.deepStrictEqual(
  numberTokens.map((t) => t.value),
  ['0xFF', '0b1010', '0o777', '3.14', '1e10'],
  'parsed number literal values must match'
);

// 4. Tokenize handles single-line and block comments with closed status
const lineCommentCode = '// single line comment\nlet y = 1;';
const lineTokens = tokenize(lineCommentCode);
const lineCommentToken = lineTokens.find((t) => t.type === 'comment');
assert.ok(lineCommentToken, 'line comment token must be found');
assert.strictEqual(lineCommentToken.value, '// single line comment');
assert.strictEqual(lineCommentToken.closed, undefined, 'line comment closed must be undefined');

const closedBlockCode = '/* multi\nline\ncomment */';
const closedBlockTokens = tokenize(closedBlockCode);
assert.strictEqual(closedBlockTokens.length, 1);
assert.strictEqual(closedBlockTokens[0].type, 'comment');
assert.strictEqual(closedBlockTokens[0].closed, true, 'closed block comment must have closed: true');

const unclosedBlockCode = '/* unclosed block comment';
const unclosedBlockTokens = tokenize(unclosedBlockCode);
assert.strictEqual(unclosedBlockTokens.length, 1);
assert.strictEqual(unclosedBlockTokens[0].type, 'comment');
assert.strictEqual(unclosedBlockTokens[0].closed, false, 'unclosed block comment must have closed: false');

// 5. Tokenize parses strings (single, double, template) and tracks closed status
const closedStringCode = '"double" \'single\' `template ${val}`';
const stringTokens = getTokensByType(closedStringCode, 'string');
assert.strictEqual(stringTokens.length, 3, 'must find 3 string tokens');
assert.strictEqual(stringTokens[0].closed, true, 'closed double-quote string closed: true');
assert.strictEqual(stringTokens[1].closed, true, 'closed single-quote string closed: true');
assert.strictEqual(stringTokens[2].closed, true, 'closed template literal string closed: true');

const unclosedStringCode = '"unclosed string literal';
const unclosedStringTokens = tokenize(unclosedStringCode);
assert.strictEqual(unclosedStringTokens.length, 1);
assert.strictEqual(unclosedStringTokens[0].type, 'string');
assert.strictEqual(unclosedStringTokens[0].closed, false, 'unclosed string must have closed: false');

// 6. Tokenize parses regular expression literals with flags
const regexCode = 'const re = /^[a-z0-9_-]+$/gi;';
const regexTokens = getTokensByType(regexCode, 'regex');
assert.strictEqual(regexTokens.length, 1, 'must find 1 regex token');
assert.strictEqual(regexTokens[0].value, '/^[a-z0-9_-]+$/gi');
assert.strictEqual(regexTokens[0].closed, undefined);

console.log('Contract tests passed successfully.');