acorn 8.14.0: tokTypes

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify acorn.tokTypes in pkg:npm/acorn@8.14.0
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/acorn@8.14.0
Demonstrate these symbols/APIs:
  - acorn.tokTypes
Constraints:
  - executionContext: node
Required runtime conditions:
  - ecosystem: npm
  - language: javascript
  - moduleSystem: cjs
  - packageManager: npm@10.9.8
  - runtime: node@22.23.2

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:c97c35b2a360e1ef2918317787b3d32770c28b5267ba92361c8ed407aa7ab658","constraints":{"executionContext":"node"},"contract":["tokTypes exposes standard literal token definitions with expression start flags","tokTypes exposes keyword token definitions with keyword strings and loop attributes","tokTypes provides operator and punctuation definitions with assignment and precedence properties","tokenizer produces tokens matching exact tokTypes singleton references","categorizeToken classifies token streams into syntax categories via tokTypes","inspectTokenType extracts normalized grammar metadata from tokTypes"],"goal":"verify acorn.tokTypes in pkg:npm/acorn@8.14.0","kind":"HOW","packages":["pkg:npm/acorn@8.14.0"],"schemaVersion":1,"symbols":["acorn.tokTypes"]},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","executionContext":"node","language":"javascript","libc":"glibc","libcVersion":"2.39","moduleSystem":"cjs","os":"linux","osVersionBucket":"24","packageManager":"npm","packageManagerVersion":"10.9.8","runtime":"node","runtimeVersion":"22.23.2","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/acorn@8.14.0"],"schemaVersion":1,"subject":"pkg:npm/acorn@8.14.0","symbols":["acorn.tokTypes"],"verifierAdapter":"node-typescript@1"}

{
  "name": "sample-acorn-toktypes",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-acorn-toktypes",
      "version": "1.0.0",
      "dependencies": {
        "acorn": "8.14.0"
      }
    },
    "node_modules/acorn": {
      "version": "8.14.0",
      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz",
      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
      "license": "MIT",
      "bin": {
        "acorn": "bin/acorn"
      },
      "engines": {
        "node": ">=0.4.0"
      }
    }
  }
}

{
  "name": "sample-acorn-toktypes",
  "version": "1.0.0",
  "private": true,
  "description": "Clean-room verification of acorn.tokTypes",
  "main": "src/index.js",
  "dependencies": {
    "acorn": "8.14.0"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify acorn.tokTypes in pkg:npm/acorn@8.14.0",
  "kind": "HOW",
  "packages": [
    "pkg:npm/acorn@8.14.0"
  ],
  "symbols": [
    "acorn.tokTypes"
  ],
  "constraints": {
    "executionContext": "node"
  },
  "runtimeConditions": {
    "ecosystem": "npm",
    "language": "javascript",
    "moduleSystem": "cjs",
    "packageManager": "npm@10.9.8",
    "runtime": "node@22.23.2"
  }
}

const acorn = require('acorn');

/**
 * Tokenize source code and return array of tokens.
 * @param {string} code - JavaScript source code to tokenize.
 * @param {object} [options] - Tokenizer options.
 * @param {boolean} [options.includeEof=false] - Whether to append the terminal EOF token.
 * @returns {Array<{ type: object, value: any, start: number, end: number }>}
 */
function tokenize(code, options = {}) {
  const { includeEof = false, ...tokenizerOptions } = options;
  const tokenizer = acorn.tokenizer(code, {
    ecmaVersion: 'latest',
    sourceType: 'module',
    ...tokenizerOptions
  });
  const tokens = [];
  for (const token of tokenizer) {
    tokens.push(token);
  }
  if (includeEof) {
    tokens.push(tokenizer.getToken());
  }
  return tokens;
}

/**
 * Categorize a token using acorn.tokTypes.
 * @param {object} token
 * @returns {'keyword' | 'literal' | 'operator' | 'punctuation' | 'identifier' | 'eof'}
 */
function categorizeToken(token) {
  const { type } = token;
  const { tokTypes } = acorn;

  if (type === tokTypes.eof) {
    return 'eof';
  }
  if (type.keyword !== undefined) {
    return 'keyword';
  }
  if (type === tokTypes.num || type === tokTypes.string || type === tokTypes.regexp) {
    return 'literal';
  }
  if (type === tokTypes.name) {
    return 'identifier';
  }
  if (type.binop !== null || type.isAssign || type.prefix || type.postfix) {
    return 'operator';
  }
  return 'punctuation';
}

/**
 * Inspect grammar attributes of a token type.
 * @param {object} tokenType
 * @returns {{ label: string, isKeyword: boolean, startsExpr: boolean, beforeExpr: boolean, isLoop: boolean, isAssign: boolean, binop: number | null }}
 */
function inspectTokenType(tokenType) {
  return {
    label: tokenType.label,
    isKeyword: tokenType.keyword !== undefined,
    startsExpr: Boolean(tokenType.startsExpr),
    beforeExpr: Boolean(tokenType.beforeExpr),
    isLoop: Boolean(tokenType.isLoop),
    isAssign: Boolean(tokenType.isAssign),
    binop: tokenType.binop ?? null
  };
}

module.exports = {
  acorn,
  tokenize,
  categorizeToken,
  inspectTokenType
};

import assert from 'node:assert';
import { createRequire } from 'node:module';

const require = createRequire(import.meta.url);
const {
  acorn,
  tokenize,
  categorizeToken,
  inspectTokenType
} = require('../src/index.js');

// 1. Export verification
assert.strictEqual(typeof acorn.tokTypes, 'object', 'acorn.tokTypes must be an object');
assert.strictEqual(typeof tokenize, 'function', 'tokenize must be a function');
assert.strictEqual(typeof categorizeToken, 'function', 'categorizeToken must be a function');
assert.strictEqual(typeof inspectTokenType, 'function', 'inspectTokenType must be a function');

// Contract 1: tokTypes exposes standard literal token definitions with expression start flags
assert.strictEqual(acorn.tokTypes.num.label, 'num');
assert.strictEqual(acorn.tokTypes.num.startsExpr, true);
assert.strictEqual(acorn.tokTypes.num.beforeExpr, false);

assert.strictEqual(acorn.tokTypes.string.label, 'string');
assert.strictEqual(acorn.tokTypes.string.startsExpr, true);
assert.strictEqual(acorn.tokTypes.string.beforeExpr, false);

assert.strictEqual(acorn.tokTypes.regexp.label, 'regexp');
assert.strictEqual(acorn.tokTypes.regexp.startsExpr, true);

// Contract 2: tokTypes exposes keyword token definitions with keyword strings and loop attributes
assert.strictEqual(acorn.tokTypes._function.keyword, 'function');
assert.strictEqual(acorn.tokTypes._function.startsExpr, true);

assert.strictEqual(acorn.tokTypes._return.keyword, 'return');
assert.strictEqual(acorn.tokTypes._return.beforeExpr, true);
assert.strictEqual(acorn.tokTypes._return.startsExpr, false);

assert.strictEqual(acorn.tokTypes._for.keyword, 'for');
assert.strictEqual(acorn.tokTypes._for.isLoop, true);
assert.strictEqual(acorn.tokTypes._while.isLoop, true);
assert.strictEqual(acorn.tokTypes._do.isLoop, true);

// Contract 3: tokTypes provides operator and punctuation definitions with assignment and precedence properties
assert.strictEqual(acorn.tokTypes.parenL.label, '(');
assert.strictEqual(acorn.tokTypes.parenL.startsExpr, true);
assert.strictEqual(acorn.tokTypes.parenL.beforeExpr, true);

assert.strictEqual(acorn.tokTypes.parenR.label, ')');
assert.strictEqual(acorn.tokTypes.parenR.startsExpr, false);

assert.strictEqual(acorn.tokTypes.eq.label, '=');
assert.strictEqual(acorn.tokTypes.eq.isAssign, true);

assert.strictEqual(typeof acorn.tokTypes.plusMin.binop, 'number');
assert.strictEqual(typeof acorn.tokTypes.star.binop, 'number');
assert(acorn.tokTypes.star.binop > acorn.tokTypes.plusMin.binop, 'multiplication precedence should be higher than addition');

// Contract 4: tokenizer produces tokens matching exact tokTypes singleton references
const code = 'const count = 10;\nreturn count + "items";';
const tokens = tokenize(code, { includeEof: true });

assert.strictEqual(tokens[0].type, acorn.tokTypes._const);
assert.strictEqual(tokens[1].type, acorn.tokTypes.name);
assert.strictEqual(tokens[2].type, acorn.tokTypes.eq);
assert.strictEqual(tokens[3].type, acorn.tokTypes.num);
assert.strictEqual(tokens[3].value, 10);
assert.strictEqual(tokens[4].type, acorn.tokTypes.semi);
assert.strictEqual(tokens[5].type, acorn.tokTypes._return);
assert.strictEqual(tokens[6].type, acorn.tokTypes.name);
assert.strictEqual(tokens[7].type, acorn.tokTypes.plusMin);
assert.strictEqual(tokens[8].type, acorn.tokTypes.string);
assert.strictEqual(tokens[8].value, 'items');
assert.strictEqual(tokens[9].type, acorn.tokTypes.semi);
assert.strictEqual(tokens[10].type, acorn.tokTypes.eof);

// Contract 5: categorizeToken classifies token streams into syntax categories via tokTypes
const categories = tokens.map(categorizeToken);
assert.deepStrictEqual(categories, [
  'keyword',      // const
  'identifier',   // count
  'operator',     // =
  'literal',      // 10
  'punctuation',  // ;
  'keyword',      // return
  'identifier',   // count
  'operator',     // +
  'literal',      // "items"
  'punctuation',  // ;
  'eof'           // eof
]);

// Contract 6: inspectTokenType extracts normalized grammar metadata from tokTypes
const numInfo = inspectTokenType(acorn.tokTypes.num);
assert.deepStrictEqual(numInfo, {
  label: 'num',
  isKeyword: false,
  startsExpr: true,
  beforeExpr: false,
  isLoop: false,
  isAssign: false,
  binop: null
});

const forInfo = inspectTokenType(acorn.tokTypes._for);
assert.deepStrictEqual(forInfo, {
  label: 'for',
  isKeyword: true,
  startsExpr: false,
  beforeExpr: false,
  isLoop: true,
  isAssign: false,
  binop: null
});

console.log('All acorn.tokTypes contract tests passed successfully.');