robots-parser 3.0.1

Clean-room public code sample — generation instructions

Write a brand-new, minimal, self-contained code sample in this clean-room directory.
Do not copy, paraphrase, or reference any existing project source. Work only from this spec.

A csx.json manifest scaffold already exists. Do not recreate it from memory. Preserve its case.goal, packages and symbols; fill its empty case.contract with exact assertions and correct its environment, commands and verifierAdapter for the files you generate.

Goal: verify pkg:npm/robots-parser@3.0.1
Kind: HOW

Use EXACTLY these public packages and versions:
  - pkg:npm/robots-parser@3.0.1

Rules:
  - One focused purpose; the smallest project that proves the goal.
  - Include a contract test (test/contract.*) that runs OFFLINE and exits 0 exactly when the goal behavior works.
  - Pin every dependency with a lockfile so resolution is reproducible.
  - No secrets, credentials, or tokens. No real URLs (only example.com or localhost). No absolute paths.
  - No personal names, emails, company names, or project identifiers of any kind.
  - No binaries and no generated output (node_modules, dist, target, venv, .git, .env).
  - Keep it under 200 files and 256KB packed.

{"case":{"caseId":"case:sha256:eb060b37af36d34034772dbbb624813d615b1e442e9ea1037f0255bc989c8266","contract":["robotsParser correctly identifies disallowed paths and returns false for isAllowed","robotsParser allows paths matching specific Allow directives even within disallowed prefixes","robotsParser prioritizes user-agent specific rules over wildcard user-agent rules","robotsParser retrieves crawl delay for specific user agents","robotsParser extracts sitemap URLs and preferred host directives","robotsParser returns 1-based matching line number or -1 when no directive matches","robotsParser returns undefined from isAllowed when the URL origin does not match the robots.txt base URL"],"goal":"verify pkg:npm/robots-parser@3.0.1","kind":"HOW","packages":["pkg:npm/robots-parser@3.0.1"],"schemaVersion":1},"contractCommand":["node","test/contract.mjs"],"environment":{"arch":"x64","distro":"ubuntu","ecosystem":"npm","libc":"glibc","libcVersion":"2.39","os":"linux","osVersionBucket":"24","packageManager":"npm","schemaVersion":1},"license":"MIT-0","packages":["pkg:npm/robots-parser@3.0.1"],"schemaVersion":1,"subject":"pkg:npm/robots-parser@3.0.1","verifierAdapter":"node-typescript@1"}

import robotsParser from 'robots-parser';

/**
 * Creates a robots parser instance for the specified base URL and robots.txt contents.
 *
 * @param {string} url - The URL where the robots.txt is hosted.
 * @param {string} contents - The robots.txt contents.
 * @returns {Object} Robots parser instance.
 */
export function createRobotsParser(url, contents) {
  return robotsParser(url, contents);
}

export default robotsParser;

{
  "name": "sample-robots-parser",
  "version": "1.0.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "sample-robots-parser",
      "version": "1.0.0",
      "dependencies": {
        "robots-parser": "3.0.1"
      }
    },
    "node_modules/robots-parser": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-3.0.1.tgz",
      "integrity": "sha512-s+pyvQeIKIZ0dx5iJiQk1tPLJAWln39+MI5jtM8wnyws+G5azk+dMnMX0qfbqNetKKNgcWWOdi0sfm+FbQbgdQ==",
      "license": "MIT",
      "engines": {
        "node": ">=10.0.0"
      }
    }
  }
}

{
  "name": "sample-robots-parser",
  "version": "1.0.0",
  "type": "module",
  "main": "index.mjs",
  "scripts": {
    "test": "node test/contract.mjs"
  },
  "dependencies": {
    "robots-parser": "3.0.1"
  }
}

{
  "schemaVersion": 1,
  "goal": "verify pkg:npm/robots-parser@3.0.1",
  "kind": "HOW",
  "packages": [
    "pkg:npm/robots-parser@3.0.1"
  ]
}

import assert from 'node:assert';
import robotsParser, { createRobotsParser } from '../index.js';

const robotsTxt = [
  'User-agent: custombot',
  'Disallow: /restricted/',
  'Allow: /restricted/public/',
  'Crawl-delay: 2.5',
  '',
  'User-agent: *',
  'Disallow: /admin/',
  'Crawl-delay: 10',
  'Sitemap: https://example.com/sitemap.xml',
  'Host: example.com'
].join('\n');

const baseUrl = 'https://example.com/robots.txt';
const parser = createRobotsParser(baseUrl, robotsTxt);

// 1. robotsParser correctly identifies disallowed paths and returns false for isAllowed
{
  const allowed = parser.isAllowed('https://example.com/admin/settings', 'otherbot');
  const disallowed = parser.isDisallowed('https://example.com/admin/settings', 'otherbot');
  assert.strictEqual(allowed, false);
  assert.strictEqual(disallowed, true);

  const allowedRoot = parser.isAllowed('https://example.com/page', 'otherbot');
  assert.strictEqual(allowedRoot, true);
}

// 2. robotsParser allows paths matching specific Allow directives even within disallowed prefixes
{
  const disallowedSub = parser.isAllowed('https://example.com/restricted/secret', 'custombot');
  assert.strictEqual(disallowedSub, false);

  const allowedSub = parser.isAllowed('https://example.com/restricted/public/file.html', 'custombot');
  assert.strictEqual(allowedSub, true);
}

// 3. robotsParser prioritizes user-agent specific rules over wildcard user-agent rules
{
  // custombot has specific rules, so /admin/ is not disallowed for custombot
  const custombotAdmin = parser.isAllowed('https://example.com/admin/settings', 'custombot');
  assert.strictEqual(custombotAdmin, true);

  // generic bot falls back to * section where /admin/ is disallowed
  const genericAdmin = parser.isAllowed('https://example.com/admin/settings', 'randombot');
  assert.strictEqual(genericAdmin, false);
}

// 4. robotsParser retrieves crawl delay for specific user agents
{
  assert.strictEqual(parser.getCrawlDelay('custombot'), 2.5);
  assert.strictEqual(parser.getCrawlDelay('otherbot'), 10);
  assert.strictEqual(parser.getCrawlDelay(), 10);
}

// 5. robotsParser extracts sitemap URLs and preferred host directives
{
  const sitemaps = parser.getSitemaps();
  assert.deepStrictEqual(sitemaps, ['https://example.com/sitemap.xml']);

  const host = parser.getPreferredHost();
  assert.strictEqual(host, 'example.com');
}

// 6. robotsParser returns 1-based matching line number or -1 when no directive matches
{
  // Line 2 is 'Disallow: /restricted/'
  const line2 = parser.getMatchingLineNumber('https://example.com/restricted/secret', 'custombot');
  assert.strictEqual(line2, 2);

  // Line 3 is 'Allow: /restricted/public/'
  const line3 = parser.getMatchingLineNumber('https://example.com/restricted/public/file.html', 'custombot');
  assert.strictEqual(line3, 3);

  // Unmatched path returns -1
  const unmatched = parser.getMatchingLineNumber('https://example.com/other/path', 'custombot');
  assert.strictEqual(unmatched, -1);
}

// 7. robotsParser returns undefined from isAllowed when the URL origin does not match the robots.txt base URL
{
  const diffOrigin = parser.isAllowed('https://other.example.com/admin/settings');
  assert.strictEqual(diffOrigin, undefined);

  const diffProto = parser.isAllowed('http://example.com/admin/settings');
  assert.strictEqual(diffProto, undefined);
}

console.log('All robots-parser contract assertions passed.');