acorn 8.14.0: tokenizer

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify acorn.tokenizer in pkg:npm/acorn@8.14.0
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/acorn@8.14.0
Demonstrate these symbols/APIs:
  - acorn.tokenizer

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:08e7fd66c927c46b20b0f10a64f633743709c47f024e632e6cb1d755921fa194","contract":["acorn.tokenizer returns a tokenizer instance with getToken and iteration capability","tokenizer produces tokens with type, value, start, end, range, and location metadata","tokenizer supports comment collection via onComment callback and options","tokenizer throws SyntaxError with position and location metadata on invalid code"],"goal":"verify acorn.tokenizer in pkg:npm/acorn@8.14.0","kind":"HOW","packages":["pkg:npm/acorn@8.14.0"],"schemaVersion":1,"symbols":["acorn.tokenizer"]},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"cjs","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/acorn@8.14.0"],"schemaVersion":1,"subject":"pkg:npm/acorn@8.14.0","symbols":["acorn.tokenizer"],"verifierAdapter":"node-typescript@1"}

{
  "name": "sample-acorn-tokenizer",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-acorn-tokenizer",
      "version": "1.0.0",
      "dependencies": {
        "acorn": "8.14.0"
      }
    },
    "node_modules/acorn": {
      "version": "8.14.0",
      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz",
      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
      "license": "MIT",
      "bin": {
        "acorn": "bin/acorn"
      },
      "engines": {
        "node": ">=0.4.0"
      }
    }
  }
}

{
  "name": "sample-acorn-tokenizer",
  "version": "1.0.0",
  "private": true,
  "description": "Clean-room verification of acorn.tokenizer",
  "main": "src/index.js",
  "dependencies": {
    "acorn": "8.14.0"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify acorn.tokenizer in pkg:npm/acorn@8.14.0",
  "kind": "HOW",
  "packages": [
    "pkg:npm/acorn@8.14.0"
  ],
  "symbols": [
    "acorn.tokenizer"
  ]
}

'use strict';

const acorn = require('acorn');

/**
 * Tokenize JavaScript source code using acorn.tokenizer.
 *
 * @param {string} code - Source code to tokenize.
 * @param {object} [options] - Acorn parser/tokenizer options.
 * @returns {object} Tokenizer instance.
 */
function createTokenizer(code, options = {}) {
  const defaultOptions = {
    ecmaVersion: 'latest',
    locations: true,
    ranges: true
  };
  return acorn.tokenizer(code, { ...defaultOptions, ...options });
}

/**
 * Collect all tokens from source code into an array using iterator protocol.
 *
 * @param {string} code - Source code to tokenize.
 * @param {object} [options] - Acorn parser/tokenizer options.
 * @returns {Array<object>} Array of non-EOF tokens.
 */
function tokenizeAll(code, options = {}) {
  const tokenizer = createTokenizer(code, options);
  return Array.from(tokenizer);
}

/**
 * Safely tokenize code, capturing syntax errors if any.
 *
 * @param {string} code - Source code to tokenize.
 * @param {object} [options] - Acorn parser/tokenizer options.
 * @returns {{ tokens?: Array<object>, error?: { message: string, pos: number, loc: object } }}
 */
function tokenizeSafe(code, options = {}) {
  try {
    const tokens = tokenizeAll(code, options);
    return { tokens };
  } catch (err) {
    return {
      error: {
        message: err.message,
        pos: err.pos,
        loc: err.loc
      }
    };
  }
}

module.exports = {
  acorn,
  createTokenizer,
  tokenizeAll,
  tokenizeSafe
};

import assert from 'node:assert/strict';
import { createRequire } from 'node:module';

const require = createRequire(import.meta.url);
const { acorn, createTokenizer, tokenizeAll, tokenizeSafe } = require('../src/index.js');

// 1. Symbol and API verification
assert.equal(typeof acorn.tokenizer, 'function', 'acorn.tokenizer must be a function');
assert.equal(typeof createTokenizer, 'function', 'createTokenizer must be a function');
assert.equal(typeof tokenizeAll, 'function', 'tokenizeAll must be a function');
assert.equal(typeof tokenizeSafe, 'function', 'tokenizeSafe must be a function');

// 2. Tokenizer getToken() and manual consumption
const code1 = 'const x = 10;';
const tokenizer1 = createTokenizer(code1);
assert.equal(typeof tokenizer1.getToken, 'function', 'tokenizer must provide getToken() method');

const tokensManual = [];
let token = tokenizer1.getToken();
while (token.type !== acorn.tokTypes.eof) {
  tokensManual.push(token);
  token = tokenizer1.getToken();
}
assert.equal(token.type, acorn.tokTypes.eof, 'last token from getToken() must be eof');
assert.equal(tokensManual.length, 5, 'should produce 5 tokens before eof for "const x = 10;"');
assert.equal(tokensManual[0].type.label, 'const');
assert.equal(tokensManual[1].type.label, 'name');
assert.equal(tokensManual[1].value, 'x');
assert.equal(tokensManual[2].type.label, '=');
assert.equal(tokensManual[3].type.label, 'num');
assert.equal(tokensManual[3].value, 10);
assert.equal(tokensManual[4].type.label, ';');

// 3. Tokenizer iterator protocol and position metadata
const code2 = 'let greeting = "hello";';
const tokens = tokenizeAll(code2);
assert.equal(tokens.length, 5, 'tokenizeAll should produce 5 tokens including semicolon');

const [tLet, tName, tAssign, tStr, tSemi] = tokens;
assert.equal(tLet.type.label, 'name', 'let is a contextual keyword tokenized as name');
assert.equal(tLet.value, 'let');
assert.equal(tLet.start, 0);
assert.equal(tLet.end, 3);
assert.deepEqual(tLet.range, [0, 3]);
assert.equal(tLet.loc.start.line, 1);
assert.equal(tLet.loc.start.column, 0);
assert.equal(tLet.loc.end.line, 1);
assert.equal(tLet.loc.end.column, 3);

assert.equal(tName.value, 'greeting');
assert.equal(tAssign.type.label, '=');
assert.equal(tStr.value, 'hello');
assert.equal(tSemi.type.label, ';');

// 4. Options: comments collection via onComment
const codeWithComments = '// line comment\nconst count = 5; /* block */';
const comments = [];
const tokensWithComments = tokenizeAll(codeWithComments, {
  onComment: comments
});

assert.equal(tokensWithComments.length, 5);
assert.equal(comments.length, 2, 'should collect 2 comments');
assert.equal(comments[0].type, 'Line');
assert.equal(comments[0].value, ' line comment');
assert.equal(comments[0].loc.start.line, 1);
assert.equal(comments[1].type, 'Block');
assert.equal(comments[1].value, ' block ');
assert.equal(comments[1].loc.start.line, 2);

// 5. Syntax error handling
const invalidCode = 'const @ = 1;';
const safeResult = tokenizeSafe(invalidCode);
assert.ok(safeResult.error, 'should return error object on syntax error');
assert.equal(typeof safeResult.error.message, 'string');
assert.equal(typeof safeResult.error.pos, 'number');
assert.ok(safeResult.error.loc, 'error should contain location info');

console.log('acorn.tokenizer contract verified successfully.');