robots-parser 3.0.1: robotsParser

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify robotsParser in pkg:npm/robots-parser@3.0.1
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/robots-parser@3.0.1
Demonstrate these symbols/APIs:
  - robotsParser

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:5a66b1376afdc93951d05e16af1eb5381aff1185ef88b995ded95390f98ed5dd","contract":["robotsParser parses robots.txt and determines path allow and disallow permissions for user agents","robotsParser respects crawl-delay directives for specific and wildcard user agents","robotsParser extracts sitemap URLs and preferred host directives","robotsParser returns matching line numbers for directive rules and handles comments and wildcards","robotsParser handles relative URLs and case-insensitive user agent matching"],"goal":"verify robotsParser in pkg:npm/robots-parser@3.0.1","kind":"HOW","packages":["pkg:npm/robots-parser@3.0.1"],"schemaVersion":1,"symbols":["robotsParser"]},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","libc":"glibc","libcVersion":"2.39","os":"linux","osVersionBucket":"24","packageManager":"npm","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/robots-parser@3.0.1"],"schemaVersion":1,"subject":"pkg:npm/robots-parser@3.0.1","symbols":["robotsParser"],"verifierAdapter":"node-typescript@1"}

{
  "name": "robots-parser-sample",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "robots-parser-sample",
      "version": "1.0.0",
      "license": "MIT-0",
      "dependencies": {
        "robots-parser": "3.0.1"
      }
    },
    "node_modules/robots-parser": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz",
      "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==",
      "license": "MIT",
      "engines": {
        "node": ">=10.0.0"
      }
    }
  }
}

{
  "name": "robots-parser-sample",
  "version": "1.0.0",
  "private": true,
  "description": "Clean-room verification for robots-parser",
  "license": "MIT-0",
  "dependencies": {
    "robots-parser": "3.0.1"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify robotsParser in pkg:npm/robots-parser@3.0.1",
  "kind": "HOW",
  "packages": [
    "pkg:npm/robots-parser@3.0.1"
  ],
  "symbols": [
    "robotsParser"
  ]
}

import assert from 'node:assert/strict';
import robotsParser from 'robots-parser';

// 1. robotsParser parses robots.txt and determines path allow and disallow permissions for user agents
const robotsTxt = [
  'User-agent: *',
  'Crawl-delay: 5',
  'Disallow: /admin/',
  'Disallow: /private.html',
  'Allow: /admin/public.html',
  '',
  'User-agent: Googlebot',
  'Crawl-delay: 10',
  'Disallow: /secret/',
  'Allow: /admin/',
  '',
  'Sitemap: https://example.com/sitemap.xml',
  'Sitemap: https://example.com/sitemap-news.xml',
  'Host: example.com'
].join('\n');

const parser = robotsParser('https://example.com/robots.txt', robotsTxt);

// Test allow / disallow for wildcard user agent
assert.equal(parser.isAllowed('https://example.com/admin/public.html', 'CustomBot/1.0'), true);
assert.equal(parser.isDisallowed('https://example.com/admin/public.html', 'CustomBot/1.0'), false);
assert.equal(parser.isAllowed('https://example.com/admin/dashboard', 'CustomBot/1.0'), false);
assert.equal(parser.isDisallowed('https://example.com/admin/dashboard', 'CustomBot/1.0'), true);
assert.equal(parser.isDisallowed('https://example.com/private.html', 'CustomBot/1.0'), true);

// Specific user agent override
assert.equal(parser.isAllowed('https://example.com/admin/dashboard', 'Googlebot'), true);
assert.equal(parser.isDisallowed('https://example.com/secret/doc.html', 'Googlebot'), true);

// 2. robotsParser respects crawl-delay directives for specific and wildcard user agents
assert.equal(parser.getCrawlDelay('CustomBot'), 5);
assert.equal(parser.getCrawlDelay('Googlebot'), 10);

// 3. robotsParser extracts sitemap URLs and preferred host directives
assert.deepEqual(parser.getSitemaps(), [
  'https://example.com/sitemap.xml',
  'https://example.com/sitemap-news.xml'
]);
assert.equal(parser.getPreferredHost(), 'example.com');

// 4. robotsParser returns matching line numbers for directive rules and handles comments and wildcards
const wildcardTxt = [
  '# Global directives',
  'User-agent: *',
  'Disallow: /*.php$',
  'Allow: /index.php'
].join('\n');

const wildcardParser = robotsParser('https://example.com/robots.txt', wildcardTxt);
assert.equal(wildcardParser.isAllowed('https://example.com/index.php', 'Bot'), true);
assert.equal(wildcardParser.getMatchingLineNumber('https://example.com/index.php', 'Bot'), 4);
assert.equal(wildcardParser.isDisallowed('https://example.com/page.php', 'Bot'), true);
assert.equal(wildcardParser.getMatchingLineNumber('https://example.com/page.php', 'Bot'), 3);
assert.equal(wildcardParser.getMatchingLineNumber('https://example.com/other.html', 'Bot'), -1);

// 5. robotsParser handles relative URLs and case-insensitive user agent matching
const relativeTxt = [
  'User-Agent: SpecialBot',
  'Disallow: /hidden/'
].join('\n');

const relativeParser = robotsParser('/robots.txt', relativeTxt);
assert.equal(relativeParser.isDisallowed('/hidden/file', 'specialbot'), true);
assert.equal(relativeParser.isAllowed('/other/file', 'specialbot'), true);

console.log('Contract passed');