robots-parser 3.0.1: getCrawlDelay

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify getCrawlDelay in pkg:npm/robots-parser@3.0.1
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/robots-parser@3.0.1
Demonstrate these symbols/APIs:
  - getCrawlDelay

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:73fe6925db57d63d523a5f8600b20aac07847af1e457e3e7c5ae4120bca4e45b","contract":["getCrawlDelay returns crawl delay specified for a specific user-agent","getCrawlDelay returns wildcard crawl delay when queried for wildcard or with default parameter","getCrawlDelay falls back to wildcard crawl delay for unspecified user-agents","getCrawlDelay returns undefined when a user-agent has rules without a crawl delay directive","getCrawlDelay returns undefined when no crawl delay is defined anywhere","getCrawlDelay normalizes user-agent strings by ignoring case and stripping version identifiers","getCrawlDelay parses decimal values and ignores non-numeric crawl delay directives"],"goal":"verify getCrawlDelay in pkg:npm/robots-parser@3.0.1","kind":"HOW","packages":["pkg:npm/robots-parser@3.0.1"],"schemaVersion":1,"symbols":["getCrawlDelay"]},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","libc":"glibc","libcVersion":"2.39","os":"linux","osVersionBucket":"24","packageManager":"npm","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/robots-parser@3.0.1"],"schemaVersion":1,"subject":"pkg:npm/robots-parser@3.0.1","symbols":["getCrawlDelay"],"verifierAdapter":"node-typescript@1"}

{
  "name": "robots-parser-sample",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "robots-parser-sample",
      "version": "1.0.0",
      "license": "MIT-0",
      "dependencies": {
        "robots-parser": "3.0.1"
      }
    },
    "node_modules/robots-parser": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz",
      "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==",
      "license": "MIT",
      "engines": {
        "node": ">=10.0.0"
      }
    }
  }
}

{
  "name": "robots-parser-sample",
  "version": "1.0.0",
  "private": true,
  "description": "Clean-room verification for robots-parser getCrawlDelay",
  "license": "MIT-0",
  "dependencies": {
    "robots-parser": "3.0.1"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify getCrawlDelay in pkg:npm/robots-parser@3.0.1",
  "kind": "HOW",
  "packages": [
    "pkg:npm/robots-parser@3.0.1"
  ],
  "symbols": [
    "getCrawlDelay"
  ]
}

import assert from 'node:assert/strict';
import robotsParser from 'robots-parser';

// 1. getCrawlDelay returns crawl delay specified for a specific user-agent
const robots1 = robotsParser('https://example.com/robots.txt', [
  'User-agent: Googlebot',
  'Crawl-delay: 2',
  'User-agent: Bingbot',
  'Crawl-delay: 4'
].join('\n'));
assert.strictEqual(robots1.getCrawlDelay('Googlebot'), 2);
assert.strictEqual(robots1.getCrawlDelay('Bingbot'), 4);

// 2. getCrawlDelay returns wildcard crawl delay when queried for wildcard or with default parameter
const robots2 = robotsParser('https://example.com/robots.txt', [
  'User-agent: *',
  'Crawl-delay: 5'
].join('\n'));
assert.strictEqual(robots2.getCrawlDelay('*'), 5);
assert.strictEqual(robots2.getCrawlDelay(), 5);

// 3. getCrawlDelay falls back to wildcard crawl delay for unspecified user-agents
const robots3 = robotsParser('https://example.com/robots.txt', [
  'User-agent: *',
  'Crawl-delay: 7'
].join('\n'));
assert.strictEqual(robots3.getCrawlDelay('UnspecifiedBot'), 7);

// 4. getCrawlDelay returns undefined when a user-agent has rules without a crawl delay directive
const robots4 = robotsParser('https://example.com/robots.txt', [
  'User-agent: *',
  'Crawl-delay: 7',
  'User-agent: CustomBot',
  'Disallow: /admin/'
].join('\n'));
assert.strictEqual(robots4.getCrawlDelay('CustomBot'), undefined);

// 5. getCrawlDelay returns undefined when no crawl delay is defined anywhere
const robots5 = robotsParser('https://example.com/robots.txt', [
  'User-agent: *',
  'Disallow: /private/',
  'User-agent: Googlebot',
  'Disallow: /secret/'
].join('\n'));
assert.strictEqual(robots5.getCrawlDelay('Googlebot'), undefined);
assert.strictEqual(robots5.getCrawlDelay('*'), undefined);
assert.strictEqual(robots5.getCrawlDelay(), undefined);
assert.strictEqual(robots5.getCrawlDelay('UnknownBot'), undefined);

// 6. getCrawlDelay normalizes user-agent strings by ignoring case and stripping version identifiers
const robots6 = robotsParser('https://example.com/robots.txt', [
  'User-agent: MyCrawler',
  'Crawl-delay: 3'
].join('\n'));
assert.strictEqual(robots6.getCrawlDelay('MyCrawler/2.1.0'), 3);
assert.strictEqual(robots6.getCrawlDelay('mycrawler'), 3);
assert.strictEqual(robots6.getCrawlDelay('MYCRAWLER/1.0 (bot@example.com)'), 3);

// 7. getCrawlDelay parses decimal values and ignores non-numeric crawl delay directives
const robots7 = robotsParser('https://example.com/robots.txt', [
  'User-agent: PreciseBot',
  'Crawl-delay: 0.5',
  'User-agent: BrokenBot',
  'Crawl-delay: not-a-number'
].join('\n'));
assert.strictEqual(robots7.getCrawlDelay('PreciseBot'), 0.5);
assert.strictEqual(robots7.getCrawlDelay('BrokenBot'), undefined);

console.log('All contract assertions passed');