espree 10.4.0: tokenize

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify espree.tokenize in pkg:npm/espree@10.4.0
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/espree@10.4.0
Demonstrate these symbols/APIs:
  - espree.tokenize

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:854520c5d19bf798d64e87619396aa6c854591ad25a75234b17fa391c5351679","contract":["tokenize returns an array of token objects for valid JavaScript code","tokenize supports ES module syntax when sourceType is module","tokenize attaches start and end range offsets when range option is enabled","tokenize attaches line and column location metadata when loc option is enabled","tokenize collects comment tokens when comment option is enabled","tokenize parses JSX elements into JSX tokens when ecmaFeatures jsx is enabled","tokenize throws SyntaxError on invalid JavaScript token with location info"],"goal":"verify espree.tokenize in pkg:npm/espree@10.4.0","kind":"HOW","packages":["pkg:npm/espree@10.4.0"],"schemaVersion":1,"symbols":["espree.tokenize"]},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"esm","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/espree@10.4.0"],"schemaVersion":1,"subject":"pkg:npm/espree@10.4.0","symbols":["espree.tokenize"],"verifierAdapter":"node-typescript@1"}

import * as espree from 'espree';

/**
 * Tokenize JavaScript code with default or custom options.
 * @param {string} code - JavaScript source string
 * @param {object} [options] - Tokenizer options
 * @returns {Array} Array of tokens
 */
export function tokenizeCode(code, options = {}) {
  return espree.tokenize(code, {
    ecmaVersion: 'latest',
    sourceType: 'script',
    ...options,
  });
}

/**
 * Tokenize JavaScript module code into tokens.
 * @param {string} code - JavaScript module source string
 * @param {object} [options] - Tokenizer options
 * @returns {Array} Array of tokens
 */
export function tokenizeModule(code, options = {}) {
  return espree.tokenize(code, {
    ecmaVersion: 'latest',
    sourceType: 'module',
    ...options,
  });
}

/**
 * Tokenize JavaScript code with range and location metadata.
 * @param {string} code - JavaScript source string
 * @param {object} [options] - Tokenizer options
 * @returns {Array} Array of tokens with range and loc metadata
 */
export function tokenizeWithLocations(code, options = {}) {
  return espree.tokenize(code, {
    ecmaVersion: 'latest',
    loc: true,
    range: true,
    ...options,
  });
}

/**
 * Tokenize JavaScript code with attached comments.
 * @param {string} code - JavaScript source string
 * @param {object} [options] - Tokenizer options
 * @returns {Array} Array of tokens with comments property
 */
export function tokenizeWithComments(code, options = {}) {
  return espree.tokenize(code, {
    ecmaVersion: 'latest',
    comment: true,
    ...options,
  });
}

/**
 * Tokenize JSX code into tokens.
 * @param {string} code - JSX source string
 * @param {object} [options] - Tokenizer options
 * @returns {Array} Array of tokens including JSX tokens
 */
export function tokenizeJSX(code, options = {}) {
  const { ecmaFeatures, ...rest } = options;
  return espree.tokenize(code, {
    ecmaVersion: 'latest',
    sourceType: 'module',
    ecmaFeatures: {
      jsx: true,
      ...ecmaFeatures,
    },
    ...rest,
  });
}

export { espree };

{
  "name": "sample-espree-tokenize",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-espree-tokenize",
      "version": "1.0.0",
      "dependencies": {
        "espree": "10.4.0"
      }
    },
    "node_modules/acorn": {
      "version": "8.18.0",
      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz",
      "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==",
      "license": "MIT",
      "bin": {
        "acorn": "bin/acorn"
      },
      "engines": {
        "node": ">=0.4.0"
      }
    },
    "node_modules/acorn-jsx": {
      "version": "5.3.2",
      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
      "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
      "license": "MIT",
      "peerDependencies": {
        "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
      }
    },
    "node_modules/eslint-visitor-keys": {
      "version": "4.2.1",
      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz",
      "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==",
      "license": "Apache-2.0",
      "engines": {
        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
      },
      "funding": {
        "url": "https://opencollective.com/eslint"
      }
    },
    "node_modules/espree": {
      "version": "10.4.0",
      "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz",
      "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==",
      "license": "BSD-2-Clause",
      "dependencies": {
        "acorn": "^8.15.0",
        "acorn-jsx": "^5.3.2",
        "eslint-visitor-keys": "^4.2.1"
      },
      "engines": {
        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
      },
      "funding": {
        "url": "https://opencollective.com/eslint"
      }
    }
  }
}

{
  "name": "sample-espree-tokenize",
  "version": "1.0.0",
  "type": "module",
  "private": true,
  "dependencies": {
    "espree": "10.4.0"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify espree.tokenize in pkg:npm/espree@10.4.0",
  "kind": "HOW",
  "packages": [
    "pkg:npm/espree@10.4.0"
  ],
  "symbols": [
    "espree.tokenize"
  ]
}

import assert from 'node:assert/strict';
import {
  espree,
  tokenizeCode,
  tokenizeModule,
  tokenizeWithLocations,
  tokenizeWithComments,
  tokenizeJSX,
} from '../index.js';

// 1. tokenize returns an array of token objects for valid JavaScript code
const basicTokens = tokenizeCode('const answer = 42;');
assert.ok(Array.isArray(basicTokens), 'tokenize must return an array');
assert.strictEqual(basicTokens.length, 5);
assert.strictEqual(basicTokens[0].type, 'Keyword');
assert.strictEqual(basicTokens[0].value, 'const');
assert.strictEqual(basicTokens[1].type, 'Identifier');
assert.strictEqual(basicTokens[1].value, 'answer');
assert.strictEqual(basicTokens[2].type, 'Punctuator');
assert.strictEqual(basicTokens[2].value, '=');
assert.strictEqual(basicTokens[3].type, 'Numeric');
assert.strictEqual(basicTokens[3].value, '42');
assert.strictEqual(basicTokens[4].type, 'Punctuator');
assert.strictEqual(basicTokens[4].value, ';');

// 2. tokenize supports ES module syntax when sourceType is module
const moduleTokens = tokenizeModule('import { add } from "./math.js";');
assert.ok(Array.isArray(moduleTokens));
assert.strictEqual(moduleTokens[0].type, 'Keyword');
assert.strictEqual(moduleTokens[0].value, 'import');
assert.strictEqual(moduleTokens[1].type, 'Punctuator');
assert.strictEqual(moduleTokens[1].value, '{');
assert.strictEqual(moduleTokens[2].type, 'Identifier');
assert.strictEqual(moduleTokens[2].value, 'add');
assert.strictEqual(moduleTokens[3].type, 'Punctuator');
assert.strictEqual(moduleTokens[3].value, '}');
assert.strictEqual(moduleTokens[4].type, 'Identifier');
assert.strictEqual(moduleTokens[4].value, 'from');
assert.strictEqual(moduleTokens[5].type, 'String');
assert.strictEqual(moduleTokens[5].value, '"./math.js"');

// 3. tokenize attaches start and end range offsets when range option is enabled
const rangeTokens = tokenizeWithLocations('let count = 0;');
assert.ok(Array.isArray(rangeTokens[0].range), 'Token range must be an array');
assert.strictEqual(rangeTokens[0].range[0], 0);
assert.strictEqual(rangeTokens[0].range[1], 3);

// 4. tokenize attaches line and column location metadata when loc option is enabled
assert.strictEqual(typeof rangeTokens[0].loc, 'object', 'Token loc must be an object');
assert.strictEqual(rangeTokens[0].loc.start.line, 1);
assert.strictEqual(rangeTokens[0].loc.start.column, 0);
assert.strictEqual(rangeTokens[0].loc.end.line, 1);
assert.strictEqual(rangeTokens[0].loc.end.column, 3);

// 5. tokenize collects comment tokens when comment option is enabled
const commentTokens = tokenizeWithComments('// header comment\nlet x = 1;');
assert.ok(Array.isArray(commentTokens.comments), 'comments property on token array must be an array');
assert.strictEqual(commentTokens.comments.length, 1);
assert.strictEqual(commentTokens.comments[0].type, 'Line');
assert.strictEqual(commentTokens.comments[0].value.trim(), 'header comment');

// 6. tokenize parses JSX elements into JSX tokens when ecmaFeatures jsx is enabled
const jsxTokens = tokenizeJSX('<Button />');
assert.ok(Array.isArray(jsxTokens));
assert.strictEqual(jsxTokens[0].type, 'Punctuator');
assert.strictEqual(jsxTokens[0].value, '<');
assert.strictEqual(jsxTokens[1].type, 'JSXIdentifier');
assert.strictEqual(jsxTokens[1].value, 'Button');
assert.strictEqual(jsxTokens[2].type, 'Punctuator');
assert.strictEqual(jsxTokens[2].value, '/');
assert.strictEqual(jsxTokens[3].type, 'Punctuator');
assert.strictEqual(jsxTokens[3].value, '>');

// 7. tokenize throws SyntaxError on invalid JavaScript token with location info
assert.throws(() => {
  espree.tokenize('"unterminated string', { ecmaVersion: 'latest' });
}, (err) => {
  return (
    err instanceof SyntaxError &&
    typeof err.lineNumber === 'number' &&
    typeof err.column === 'number' &&
    typeof err.index === 'number'
  );
}, 'Invalid JS token must throw SyntaxError with lineNumber, column, and index');

console.log('Contract verification finished successfully: all espree.tokenize assertions passed.');