edufeed/amb-sitemap-parser
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

lots of improvements, including publishing pipeline

Browse source
This commit is contained in:
@s.roertgen 2026-01-30 13:05:37 +01:00
commit a34c6c8268
14 changed files with 1061 additions and 281 deletions
Download patch file Download diff file Expand all files Collapse all files

29
.forgejo/workflows/publish-dev.yml Normal file
View file

@ -0,0 +1,29 @@
name: Publish Dev Build
on:
push:
branches: [main]
jobs:
publish:
runs-on: docker
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '20'
- run: npm ci
- run: npm test
- run: npm run build
- name: Set dev version
run: |
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
npm version "0.0.0-${SHORT_SHA}" --no-git-tag-version
- name: Configure registry auth
run: |
npm config set -- '//git.edufeed.org/api/packages/edufeed/npm/:_authToken' "${{ secrets.REGISTRY_TOKEN }}"
- name: Publish with dev tag
run: npm publish --tag dev --registry=https://git.edufeed.org/api/packages/edufeed/npm/

116
CLAUDE.md Normal file
View file

@ -0,0 +1,116 @@
# CLAUDE.md
## Project Overview
**amb-sitemap-parser** is a CLI tool and npm library for parsing sitemaps and extracting educational metadata (JSON-LD) from web pages following the AMB (Allgemeines Metadatenprofil für Bildungsressourcen) standard.
## Tech Stack
- **Runtime**: Node.js >= 20.0.0
- **Language**: TypeScript 5.9.3
- **Module System**: Pure ESM (`"type": "module"`)
- **Dependencies**: cheerio (HTML), fast-xml-parser (XML), commander (CLI)
- **Testing**: Vitest
## Build & Development Commands
```bash
npm run build # Compile TypeScript
npm run dev # Watch mode
npm run test # Run tests
npm run test:watch # Test watch mode
npm run test:coverage # Coverage report
npm run lint # Check code style
npm run lint:fix # Auto-fix linting
npm run format # Format with Prettier
npm run cli # Run CLI from dist/
```
## Project Structure
```
src/
├── cli.ts # CLI interface (commander)
├── index.ts # Library entry point
├── types.ts # TypeScript interfaces
├── SitemapParser.ts # XML sitemap parsing
├── PageFetcher.ts # HTTP fetching with concurrency
├── MetadataExtractor.ts # JSON-LD extraction from HTML
├── outputWriter.ts # JSONL file output
└── progress.ts # TTY-aware progress indicator
```
## Architecture
Pipeline pattern: `Sitemap → Parse → Fetch → Extract → Filter`
Three main classes with single responsibilities:
- **SitemapParser**: XML → URLs
- **PageFetcher**: URLs → HTML (concurrent, rate-limited)
- **MetadataExtractor**: HTML → JSON-LD metadata
## Key Patterns
### ESM Imports
Always use `.js` extension in imports (even for `.ts` files):
```typescript
import { SitemapParser } from './SitemapParser.js';
```
### Error Handling
Never throw in batch operations. Capture errors in result objects:
```typescript
interface FetchResult {
url: string;
success: boolean;
error?: string; // Error captured here, not thrown
}
```
### Logger Injection
All classes accept optional logger for debugging:
```typescript
const parser = new SitemapParser({
logger: (msg, level) => console.error(`[${level}] ${msg}`)
});
```
### Async/Await
Use async/await throughout, no callbacks or .then() chains.
## CLI Commands
```bash
# Parse sitemap(s), list URLs
amb-sitemap-parser parse <sitemap-url...> [--limit N] [--pretty] [--insecure]
# Full extraction pipeline
amb-sitemap-parser extract <sitemap-url...> [--limit N] [--max-concurrency N] [--timeout N] [--verbose] [--quiet] [--jsonl] [--output file.jsonl] [--insecure]
# Direct URL fetch (no sitemap)
amb-sitemap-parser fetch <url...> [--verbose] [--quiet] [--jsonl] [--output file.jsonl] [--insecure]
```
### JSONL Streaming
`--jsonl` outputs one AMB resource per line to stdout for piping.
`--output` writes JSONL to file.
These options are mutually exclusive.
## Educational Content Detection
Identifies educational content by checking JSON-LD `@type` for: `LearningResource`, `Course`, `CreativeWork`, `Article`, or types containing `Educational`. Also supports AMB object types (`{id: "...LearningResource..."}`). Supports string or array `@type` values.
## Integration with amb-nostr-converter
`amb-convert` accepts JSONL input natively, so pipe or file-based workflows are straightforward:
```bash
amb-sitemap-parser extract url -o resources.jsonl
cat resources.jsonl | amb-convert amb:nostr --nsec $NOSTR_NSEC -o events.jsonl
```
## Code Quality
- TypeScript strict mode enabled
- ESLint + Prettier configured
- No `any` types except for dynamic JSON-LD data
- Explicit return types on public methods

257
README.md
View file

@ -4,13 +4,13 @@ A modern TypeScript library for parsing sitemaps and extracting educational meta
## Features
- 📋 Parse XML sitemaps (regular sitemaps and sitemap indexes)
- 🌐 Concurrent page fetching with rate limiting
- 🔍 Extract JSON-LD metadata from HTML pages
- 📊 Filter educational content automatically
- 🎯 Full TypeScript support with exported types
- 🚀 Pure ESM for modern JavaScript environments
- 🧪 Tested with Vitest
- Parse XML sitemaps (regular sitemaps and sitemap indexes)
- Concurrent page fetching with rate limiting
- Extract JSON-LD metadata from HTML pages
- Filter educational content automatically
- Full TypeScript support with exported types
- Pure ESM for modern JavaScript environments
- Tested with Vitest
## Installation
@ -172,21 +172,28 @@ amb-sitemap-parser fetch https://example.com/course > result.json
#### `parse` command options:
- `-l, --limit <number>` - Limit to first N URLs
- `-p, --pretty` - Pretty-print JSON output
- `-k, --insecure` - Skip SSL certificate verification
#### `extract` command options:
- `-l, --limit <number>` - Limit URLs to process
- `-c, --max-concurrency <number>` - Maximum concurrent requests (default: 5)
- `-t, --timeout <number>` - Request timeout in milliseconds (default: 30000)
- `-o, --output <filepath>` - Save metadata to JSONL file (one JSON-LD object per line)
- `--jsonl` - Stream one JSON-LD resource per line to stdout
- `-p, --pretty` - Pretty-print JSON output
- `-v, --verbose` - Show progress logs (written to stderr)
- `-q, --quiet` - Suppress progress output
- `-k, --insecure` - Skip SSL certificate verification
#### `fetch` command options:
- `-c, --max-concurrency <number>` - Maximum concurrent requests (default: 5)
- `-t, --timeout <number>` - Request timeout in milliseconds (default: 30000)
- `-o, --output <filepath>` - Save metadata to JSONL file (one JSON-LD object per line)
- `--jsonl` - Stream one JSON-LD resource per line to stdout
- `-p, --pretty` - Pretty-print JSON output
- `-v, --verbose` - Show progress logs (written to stderr)
- `-q, --quiet` - Suppress progress output
- `-k, --insecure` - Skip SSL certificate verification
### JSONL Output Format
@ -197,17 +204,12 @@ When using the `--output` option, metadata is saved in **JSON Lines (JSONL)** fo
Each JSON-LD object from a page becomes a separate line in the file:
```jsonl
{"url":"https://example.com/course1","jsonLd":{"@type":"Course","name":"Python 101"}}
{"url":"https://example.com/course1","jsonLd":{"@type":"LearningResource","name":"Exercise 1"}}
{"url":"https://example.com/course2","jsonLd":{"@type":"Course","name":"JavaScript Basics"}}
{"@type":"Course","name":"Python 101","url":"https://example.com/course1"}
{"@type":"LearningResource","name":"Exercise 1","url":"https://example.com/course1"}
{"@type":"Course","name":"JavaScript Basics","url":"https://example.com/course2"}
```
**Key benefits:**
- ✅ Each line is a complete, independent record
- ✅ Stream-friendly (process one record at a time)
- ✅ Perfect for Unix pipelines (`jq`, `grep`, `awk`)
- ✅ Easy to parallelize processing
- ✅ If a page has multiple JSON-LD objects, each gets its own line
Each line is a complete, independent record. If a page has multiple JSON-LD objects, each gets its own line. This format is ideal for streaming and Unix pipelines (`jq`, `grep`, `awk`).
#### Output Behavior
@ -259,219 +261,18 @@ amb-sitemap-parser fetch https://example.com/course --output course.jsonl
amb-sitemap-parser extract https://example.com/sitemap.xml | jq '.metadata[] | select(.jsonLd != null)'
```
### JSONL Piping Examples
The JSONL format enables powerful command-line workflows:
```bash
# Save metadata to file
amb-sitemap-parser extract https://example.com/sitemap.xml -o results.jsonl -v
# Count total records
wc -l < results.jsonl
# Filter by type: only Courses
cat results.jsonl | jq 'select(.jsonLd."@type" == "Course")'
# Extract all course names
cat results.jsonl | jq -r '.jsonLd.name' | sort | uniq
# Count by type
cat results.jsonl | jq -r '.jsonLd."@type"' | sort | uniq -c
# Find courses from specific domain
cat results.jsonl | jq 'select(.url | contains("example.com"))'
# Get URLs with specific property
cat results.jsonl | jq -r 'select(.jsonLd.educationalLevel) | .url'
# Process one record at a time
cat results.jsonl | while read line; do
echo "$line" | jq '.jsonLd.name'
done
# Parallel processing with GNU parallel
cat results.jsonl | parallel --pipe -L1 'echo {} | jq ".jsonLd.name"'
# Convert to CSV (name and URL only)
cat results.jsonl | jq -r '[.jsonLd.name, .url] | @csv'
# Filter and save to new file
cat results.jsonl | jq 'select(.jsonLd."@type" == "Course")' > courses-only.jsonl
```
## Integration with AMB-Nostr-Converter
Convert extracted AMB resources directly to Nostr events using the `--jsonl` flag for seamless streaming between tools.
### Prerequisites
Install the AMB-Nostr converter:
Extract AMB resources and convert to Nostr events with [`amb-convert`](https://www.npmjs.com/package/@edufeed-org/amb-nostr-converter):
```bash
npm install -g @edufeed-org/amb-nostr-converter
# Or use npx for on-demand usage
```
# Extract to JSONL, then convert to signed Nostr events
amb-sitemap-parser extract https://example.com/sitemap.xml -o resources.jsonl
cat resources.jsonl | amb-convert amb:nostr --nsec $NOSTR_NSEC -o events.jsonl
### Basic Pipeline
The `--jsonl` flag outputs one AMB resource per line to stdout, perfect for piping:
```bash
# Extract and convert a single resource
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl --limit 1 | \
npx amb-convert amb:nostr --pretty
# Extract from direct URL
amb-sitemap-parser fetch https://example.com/course --jsonl | \
npx amb-convert amb:nostr --pretty
```
### Processing Multiple Resources
Convert all resources from a sitemap to Nostr events:
> ⚠️ **Important:** When using a `while read` loop with piped input, `amb-convert` can consume stdin from the outer pipeline, causing most lines to be skipped. **Always save to a temp file first**, then process:
```bash
# ✅ CORRECT: Save to temp file first, then process
TEMP_FILE=$(mktemp)
trap "rm -f $TEMP_FILE" EXIT
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl > "$TEMP_FILE"
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr 2>/dev/null
done < "$TEMP_FILE" > nostr-events.jsonl
# ❌ WRONG: Direct piping loses most lines due to stdin consumption
# amb-sitemap-parser extract url --jsonl | while read resource; do ...
```
**Why does this happen?** When `amb-convert` runs inside the `while read` loop, it reads from stdin and inadvertently consumes input from the outer pipeline that was meant for the `read` command. This causes most lines to be skipped.
**With verbose logging:**
```bash
TEMP_FILE=$(mktemp)
trap "rm -f $TEMP_FILE" EXIT
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl -v 2>&1 | tee /dev/stderr > "$TEMP_FILE"
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr 2>/dev/null
done < "$TEMP_FILE" > nostr-events.jsonl
```
### Signing Nostr Events
Sign events with your Nostr private key:
```bash
# Save to temp file first (required to avoid stdin consumption issues)
TEMP_FILE=$(mktemp)
trap "rm -f $TEMP_FILE" EXIT
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl > "$TEMP_FILE"
# Using nsec (bech32 format)
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC 2>/dev/null
done < "$TEMP_FILE" > signed-events.jsonl
# Using hex private key
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr --private-key $NOSTR_PRIVATE_KEY 2>/dev/null
done < "$TEMP_FILE" > signed-events.jsonl
```
### Complete Workflow Examples
```bash
# 1. Test with a single URL first (single resource is OK to pipe directly)
amb-sitemap-parser fetch https://example.com/course --jsonl -v | \
npx amb-convert amb:nostr --pretty
# 2. Process multiple resources (MUST use temp file approach)
TEMP_FILE=$(mktemp)
trap "rm -f $TEMP_FILE" EXIT
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl -l 10 -v > "$TEMP_FILE"
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC 2>/dev/null
done < "$TEMP_FILE" > events.jsonl
# 3. Full sitemap conversion with signed events
TEMP_FILE=$(mktemp)
trap "rm -f $TEMP_FILE" EXIT
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl --max-concurrency 10 -v 2> extraction.log > "$TEMP_FILE"
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC --pretty 2>/dev/null
done < "$TEMP_FILE" > all-events.jsonl
# 4. Filter and convert only Course types (jq doesn't consume stdin, so intermediate file is safe)
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl > amb-resources.jsonl
jq -c 'select(.type | contains(["Course"]))' amb-resources.jsonl > courses.jsonl
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr 2>/dev/null
done < courses.jsonl > courses-events.jsonl
# 5. Save AMB resources and convert separately (recommended approach)
amb-sitemap-parser extract https://example.com/sitemap.xml -o amb-resources.jsonl -v
while IFS= read -r resource; do
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC 2>/dev/null
done < amb-resources.jsonl > nostr-events.jsonl
```
### Why `--jsonl` Flag?
The `--jsonl` flag is specifically designed for tool integration:
**Without `--jsonl` (default):**
```json
{
"metadata": [
{ "url": "...", "jsonLd": [{...}] },
{ "url": "...", "jsonLd": [{...}] }
],
"summary": {...}
}
```
❌ Complex nested structure, hard to pipe
**With `--jsonl`:**
```
{AMB resource 1}
{AMB resource 2}
{AMB resource 3}
```
✅ One resource per line, perfect for streaming
### Integration Benefits
- ✅ **Streaming**: Process resources one at a time, low memory usage
- ✅ **Composable**: Combine with standard Unix tools (`jq`, `grep`, `awk`)
- ✅ **Resumable**: Stop and resume processing at any point
- ✅ **Flexible**: Filter, transform, and route data as needed
- ✅ **Observable**: Use verbose mode (`-v`) to monitor progress via stderr
### Troubleshooting
**Issue**: "Cannot use --output and --jsonl together"
```bash
# ❌ Wrong
amb-sitemap-parser extract url --output file.jsonl --jsonl
# ✅ Correct (save to file)
amb-sitemap-parser extract url --output file.jsonl
# ✅ Correct (pipe to stdout)
amb-sitemap-parser extract url --jsonl > file.jsonl
```
**Issue**: No output when piping
```bash
# Make sure to use --jsonl flag for streaming
amb-sitemap-parser extract url --jsonl | npx amb-convert amb:nostr
```
**Issue**: Need to see progress while piping
```bash
# Use --verbose flag (logs go to stderr, data to stdout)
amb-sitemap-parser extract url --jsonl -v | npx amb-convert amb:nostr 2>&1
# Or pipe directly
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl | \
amb-convert amb:nostr --nsec $NOSTR_NSEC -o events.jsonl
```
## Programmatic Usage (Node.js)
@ -673,9 +474,15 @@ npm run lint
npm run format
```
## Related Projects
- [AMB-Nostr Converter](https://git.edufeed.org/edufeed/amb-nostr-converter) - Convert AMB metadata to Nostr events (kind 30142)
- [AMB Specification](https://w3id.org/kim/amb/) - General Metadata Profile for Learning Resources
- [AMB-NIP (kind 30142)](https://github.com/edufeed-org/nips/blob/edufeed-amb/edufeed.md) - Nostr event spec for AMB
## License
MIT
Unlicense
## Contributing

13
package.json
View file

@ -35,7 +35,7 @@
"node": ">=20.0.0"
},
"scripts": {
"build": "tsc",
"build": "rm -rf dist && tsc && chmod +x dist/cli.js",
"cli": "node dist/cli.js",
"dev": "tsc --watch",
"test": "vitest run",
@ -45,7 +45,7 @@
"lint": "eslint src/**/*.ts",
"lint:fix": "eslint src/**/*.ts --fix",
"format": "prettier --write \"src/**/*.ts\"",
"prepublishOnly": "npm run test && npm run build"
"prepublishOnly": "npm run build && npm run test"
},
"keywords": [
"sitemap",
@ -59,7 +59,14 @@
"esm"
],
"author": "",
"license": "MIT",
"license": "Unlicense",
"repository": {
"type": "git",
"url": "https://git.edufeed.org/edufeed/amb-sitemap-parser.git"
},
"publishConfig": {
"registry": "https://git.edufeed.org/api/packages/edufeed/npm/"
},
"dependencies": {
"cheerio": "^1.1.2",
"commander": "^14.0.2",

143
src/MetadataExtractor.test.ts Normal file
View file

@ -0,0 +1,143 @@
import { describe, it, expect } from 'vitest';
import { MetadataExtractor } from './MetadataExtractor.js';
import type { ExtractedMetadata } from './types.js';
describe('MetadataExtractor', () => {
describe('filterEducationalMetadata', () => {
it('should handle AMB format with object types', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{
type: [{
id: 'https://example.com/vocabs/type/LearningResource',
prefLabel: { de: 'Lernressource' },
type: 'Concept'
}]
}]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(1);
});
it('should handle standard schema.org string @type', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ '@type': 'LearningResource' }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(1);
});
it('should handle array of string types', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ '@type': ['Thing', 'LearningResource'] }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(1);
});
it('should filter out non-educational content', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ '@type': 'Organization' }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(0);
});
it('should filter out metadata without jsonLd', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
name: 'Some Page'
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(0);
});
it('should handle mixed AMB and standard types', () => {
const metadata: ExtractedMetadata[] = [
{
url: 'https://example.com/amb',
extractionTime: 100,
jsonLd: [{
type: [{
id: 'https://example.com/vocabs/type/LearningResource',
prefLabel: { de: 'Lernressource' },
type: 'Concept'
}]
}]
},
{
url: 'https://example.com/standard',
extractionTime: 100,
jsonLd: [{ '@type': 'Course' }]
},
{
url: 'https://example.com/other',
extractionTime: 100,
jsonLd: [{ '@type': 'Person' }]
}
];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(2);
expect(result.map(r => r.url)).toContain('https://example.com/amb');
expect(result.map(r => r.url)).toContain('https://example.com/standard');
});
it('should detect Course type', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ '@type': 'Course' }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(1);
});
it('should detect CreativeWork type', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ '@type': 'CreativeWork' }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(1);
});
it('should detect Article type', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ '@type': 'Article' }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(1);
});
it('should handle null/undefined type gracefully', () => {
const metadata: ExtractedMetadata[] = [{
url: 'https://example.com',
extractionTime: 100,
jsonLd: [{ name: 'No type here' }]
}];
const result = MetadataExtractor.filterEducationalMetadata(metadata);
expect(result).toHaveLength(0);
});
});
});

32
src/MetadataExtractor.ts
View file

@ -197,7 +197,7 @@ export class MetadataExtractor {
/**
* Filter metadata to only include AMB-relevant data
* This looks for LearningResource, Course, or educational schema.org types
*
*
* @param metadata - Array of extracted metadata
* @returns Filtered array containing only educational content
*/
@ -211,13 +211,29 @@ export class MetadataExtractor {
const types = Array.isArray(type) ? type : [type];
return types.some((t: string) =>
t.includes('LearningResource') ||
t.includes('Course') ||
t.includes('Educational') ||
t.includes('CreativeWork') ||
t.includes('Article')
);
return types.some((t: any) => {
// Handle string types (standard schema.org)
if (typeof t === 'string') {
return (
t.includes('LearningResource') ||
t.includes('Course') ||
t.includes('Educational') ||
t.includes('CreativeWork') ||
t.includes('Article')
);
}
// Handle object types with id (AMB format)
if (t && typeof t === 'object' && typeof t.id === 'string') {
return (
t.id.includes('LearningResource') ||
t.id.includes('Course') ||
t.id.includes('Educational') ||
t.id.includes('CreativeWork') ||
t.id.includes('Article')
);
}
return false;
});
});
});
}

221
src/PageFetcher.test.ts Normal file
View file

@ -0,0 +1,221 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { PageFetcher } from './PageFetcher.js';
describe('PageFetcher', () => {
beforeEach(() => {
vi.restoreAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('SSL error handling', () => {
it('should return helpful error message for SSL certificate errors', async () => {
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('unable to verify the first certificate')
);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(false);
expect(result.error).toContain('SSL certificate verification failed');
expect(result.error).toContain('--insecure');
});
it('should include URL in SSL error message', async () => {
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('unable to verify the first certificate')
);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.error).toContain('example.com');
});
it('should detect various SSL error patterns', async () => {
const sslErrors = [
'unable to verify the first certificate',
'certificate has expired',
'self signed certificate',
'CERT_HAS_EXPIRED',
];
for (const errorMessage of sslErrors) {
vi.spyOn(global, 'fetch').mockRejectedValue(new Error(errorMessage));
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(false);
expect(result.error).toContain('SSL certificate verification failed');
vi.restoreAllMocks();
}
});
it('should detect SSL errors wrapped in error.cause (Node.js fetch behavior)', async () => {
// Node.js native fetch wraps SSL errors in a TypeError with .cause
const wrappedError = new TypeError('fetch failed');
(wrappedError as Error & { cause: Error }).cause = new Error('unable to verify the first certificate');
vi.spyOn(global, 'fetch').mockRejectedValue(wrappedError);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(false);
expect(result.error).toContain('SSL certificate verification failed');
});
it('should not retry SSL errors', async () => {
const fetchMock = vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('unable to verify the first certificate')
);
const fetcher = new PageFetcher({ retryAttempts: 3 });
await fetcher.fetchSinglePage('https://example.com/page');
// SSL errors should not trigger retries
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('should pass through non-SSL errors', async () => {
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('Network error: ECONNREFUSED')
);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(false);
expect(result.error).toBe('Network error: ECONNREFUSED');
});
});
describe('insecure mode', () => {
it('should accept insecure option in constructor', () => {
const fetcher = new PageFetcher({ insecure: true });
expect(fetcher).toBeDefined();
});
it('should pass dispatcher to fetch when insecure is true', async () => {
const mockFetch = vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('<html></html>', { status: 200 })
);
const fetcher = new PageFetcher({ insecure: true });
await fetcher.fetchSinglePage('https://example.com/page');
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/page',
expect.objectContaining({
dispatcher: expect.anything(),
})
);
});
it('should not pass dispatcher when insecure is false', async () => {
const mockFetch = vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('<html></html>', { status: 200 })
);
const fetcher = new PageFetcher({ insecure: false });
await fetcher.fetchSinglePage('https://example.com/page');
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/page',
expect.objectContaining({
dispatcher: undefined,
})
);
});
});
describe('fetchSinglePage', () => {
it('should return success result for successful fetch', async () => {
vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('<html><body>Hello</body></html>', { status: 200 })
);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(true);
expect(result.html).toBe('<html><body>Hello</body></html>');
expect(result.statusCode).toBe(200);
expect(result.responseTime).toBeDefined();
});
it('should return error result for HTTP errors', async () => {
vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('Not Found', { status: 404, statusText: 'Not Found' })
);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(false);
expect(result.error).toBe('HTTP 404: Not Found');
expect(result.statusCode).toBe(404);
});
it('should handle timeout errors', async () => {
const abortError = new Error('Aborted');
abortError.name = 'AbortError';
vi.spyOn(global, 'fetch').mockRejectedValue(abortError);
const fetcher = new PageFetcher();
const result = await fetcher.fetchSinglePage('https://example.com/page');
expect(result.success).toBe(false);
expect(result.error).toBe('Request timeout');
});
});
describe('fetchPages', () => {
it('should fetch multiple pages in batch', async () => {
// Mock fetch to return a new Response for each call
vi.spyOn(global, 'fetch').mockImplementation(() =>
Promise.resolve(new Response('<html></html>', { status: 200 }))
);
const fetcher = new PageFetcher({ maxConcurrency: 2, delayBetweenRequests: 0 });
const urls = [
{ loc: 'https://example.com/page1' },
{ loc: 'https://example.com/page2' },
{ loc: 'https://example.com/page3' },
];
const results = await fetcher.fetchPages(urls);
expect(results).toHaveLength(3);
results.forEach((result, index) => {
if (!result.success) {
console.error(`Result ${index} failed:`, result.error);
}
});
expect(results.every(r => r.success)).toBe(true);
});
it('should pass insecure option to batch fetches', async () => {
const mockFetch = vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('<html></html>', { status: 200 })
);
const fetcher = new PageFetcher({ insecure: true, maxConcurrency: 1 });
const urls = [{ loc: 'https://example.com/page1' }];
await fetcher.fetchPages(urls);
expect(mockFetch).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({
dispatcher: expect.anything(),
})
);
});
});
});

78
src/PageFetcher.ts
View file

@ -1,9 +1,51 @@
import { Agent } from 'undici';
import type { SitemapUrl, FetchResult, FetchOptions, LoggerFunction } from './types.js';
export interface PageFetcherOptions extends FetchOptions {
logger?: LoggerFunction;
retryAttempts?: number;
retryDelay?: number;
/** Callback for progress updates during batch fetching */
onProgress?: (completed: number, total: number) => void;
}
/**
* Check if an error is related to SSL/TLS certificate verification
* Node.js fetch wraps SSL errors in a TypeError with the actual error in .cause
*/
function isSSLError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const sslErrorPatterns = [
'unable to verify the first certificate',
'certificate has expired',
'self signed certificate',
'CERT_',
'SSL_',
'UNABLE_TO_GET_ISSUER_CERT',
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
'DEPTH_ZERO_SELF_SIGNED_CERT',
];
// Check the error itself
const message = error.message || '';
const code = (error as NodeJS.ErrnoException).code || '';
if (sslErrorPatterns.some(pattern => message.includes(pattern) || code.includes(pattern))) {
return true;
}
// Check error.cause (Node.js fetch wraps SSL errors this way)
const cause = (error as Error & { cause?: Error }).cause;
if (cause instanceof Error) {
const causeMessage = cause.message || '';
const causeCode = (cause as NodeJS.ErrnoException).code || '';
if (sslErrorPatterns.some(pattern => causeMessage.includes(pattern) || causeCode.includes(pattern))) {
return true;
}
}
return false;
}
/**
@ -16,20 +58,25 @@ export interface PageFetcherOptions extends FetchOptions {
* ```
*/
export class PageFetcher {
private defaultOptions: Required<Omit<PageFetcherOptions, 'logger' | 'retryAttempts' | 'retryDelay'>>;
private defaultOptions: Required<Omit<PageFetcherOptions, 'logger' | 'retryAttempts' | 'retryDelay' | 'onProgress'>>;
private logger?: LoggerFunction;
private retryAttempts: number;
private retryDelay: number;
private insecure: boolean;
private onProgress?: (completed: number, total: number) => void;
constructor(options: PageFetcherOptions = {}) {
this.logger = options.logger;
this.retryAttempts = options.retryAttempts ?? 0;
this.retryDelay = options.retryDelay ?? 1000;
this.insecure = options.insecure ?? false;
this.onProgress = options.onProgress;
this.defaultOptions = {
timeout: options.timeout ?? 30000,
maxConcurrency: options.maxConcurrency ?? 5,
userAgent: options.userAgent ?? 'amb-sitemap-parser/0.1.0 (Educational Research)',
delayBetweenRequests: options.delayBetweenRequests ?? 100,
insecure: options.insecure ?? false,
};
}
@ -61,6 +108,9 @@ export class PageFetcher {
const successfulInBatch = batchResults.filter(r => r.success).length;
this.logger?.(`Batch ${batchIndex} complete: ${successfulInBatch}/${batch.length} successful`, 'info');
// Report progress
this.onProgress?.(results.length, urls.length);
} catch (error) {
this.logger?.(`Batch ${batchIndex} failed: ${error}`, 'error');
// Add failed results for all URLs in this batch
@ -70,6 +120,9 @@ export class PageFetcher {
error: 'Batch processing failed',
}));
results.push(...failedResults);
// Report progress even on failure
this.onProgress?.(results.length, urls.length);
}
// Add delay between batches if not the last batch
@ -85,7 +138,7 @@ export class PageFetcher {
/**
* Fetch a single page with timeout and error handling
*
*
* @param url - The URL to fetch
* @param options - Fetch options
* @returns Fetch result with HTML content or error
@ -94,10 +147,17 @@ export class PageFetcher {
const opts = options ?? {
timeout: this.defaultOptions.timeout,
userAgent: this.defaultOptions.userAgent,
insecure: this.insecure,
};
const startTime = Date.now();
// Create custom dispatcher to skip SSL verification if insecure mode is enabled
const useInsecure = opts.insecure ?? this.insecure;
const dispatcher = useInsecure
? new Agent({ connect: { rejectUnauthorized: false } })
: undefined;
for (let attempt = 0; attempt <= this.retryAttempts; attempt++) {
try {
const controller = new AbortController();
@ -114,6 +174,8 @@ export class PageFetcher {
'Upgrade-Insecure-Requests': '1',
},
redirect: 'follow',
// @ts-expect-error - dispatcher is a valid undici option for Node.js native fetch
dispatcher,
});
clearTimeout(timeoutId);
@ -149,6 +211,18 @@ export class PageFetcher {
} catch (error) {
const responseTime = Date.now() - startTime;
// Check for SSL errors before retry logic
if (isSSLError(error)) {
return {
url,
success: false,
error: `SSL certificate verification failed for ${url}. ` +
`The server may have an incomplete certificate chain. ` +
`Use --insecure to bypass certificate verification.`,
responseTime,
};
}
if (attempt < this.retryAttempts) {
this.logger?.(`Retry attempt ${attempt + 1}/${this.retryAttempts} for ${url}`, 'warn');
await this.delay(this.retryDelay * (attempt + 1));

193
src/SitemapParser.test.ts Normal file
View file

@ -0,0 +1,193 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { SitemapParser } from './SitemapParser.js';
describe('SitemapParser', () => {
beforeEach(() => {
vi.restoreAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('SSL error handling', () => {
it('should provide helpful error message for SSL certificate errors', async () => {
// Mock fetch to throw SSL error
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('unable to verify the first certificate')
);
const parser = new SitemapParser();
await expect(parser.parseFromUrl('https://example.com/sitemap.xml'))
.rejects.toThrow(/SSL certificate verification failed/);
});
it('should suggest --insecure flag in SSL error message', async () => {
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('unable to verify the first certificate')
);
const parser = new SitemapParser();
await expect(parser.parseFromUrl('https://example.com/sitemap.xml'))
.rejects.toThrow(/--insecure/);
});
it('should include URL in SSL error message', async () => {
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('unable to verify the first certificate')
);
const parser = new SitemapParser();
await expect(parser.parseFromUrl('https://example.com/sitemap.xml'))
.rejects.toThrow(/example\.com/);
});
it('should detect various SSL error patterns', async () => {
const sslErrors = [
'unable to verify the first certificate',
'certificate has expired',
'self signed certificate',
'CERT_HAS_EXPIRED',
];
for (const errorMessage of sslErrors) {
vi.spyOn(global, 'fetch').mockRejectedValue(new Error(errorMessage));
const parser = new SitemapParser();
await expect(parser.parseFromUrl('https://example.com/sitemap.xml'))
.rejects.toThrow(/SSL certificate verification failed/);
vi.restoreAllMocks();
}
});
it('should detect SSL errors wrapped in error.cause (Node.js fetch behavior)', async () => {
// Node.js native fetch wraps SSL errors in a TypeError with .cause
const wrappedError = new TypeError('fetch failed');
(wrappedError as Error & { cause: Error }).cause = new Error('unable to verify the first certificate');
vi.spyOn(global, 'fetch').mockRejectedValue(wrappedError);
const parser = new SitemapParser();
await expect(parser.parseFromUrl('https://example.com/sitemap.xml'))
.rejects.toThrow(/SSL certificate verification failed/);
});
it('should pass through non-SSL errors unchanged', async () => {
vi.spyOn(global, 'fetch').mockRejectedValue(
new Error('Network error: ECONNREFUSED')
);
const parser = new SitemapParser();
await expect(parser.parseFromUrl('https://example.com/sitemap.xml'))
.rejects.toThrow('Network error: ECONNREFUSED');
});
});
describe('insecure mode', () => {
it('should accept insecure option in constructor', () => {
const parser = new SitemapParser({ insecure: true });
expect(parser).toBeDefined();
});
it('should pass dispatcher to fetch when insecure is true', async () => {
const mockFetch = vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('<?xml version="1.0"?><urlset><url><loc>https://example.com</loc></url></urlset>', {
status: 200,
})
);
const parser = new SitemapParser({ insecure: true });
await parser.parseFromUrl('https://example.com/sitemap.xml');
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/sitemap.xml',
expect.objectContaining({
dispatcher: expect.anything(),
})
);
});
it('should not pass dispatcher when insecure is false', async () => {
const mockFetch = vi.spyOn(global, 'fetch').mockResolvedValue(
new Response('<?xml version="1.0"?><urlset><url><loc>https://example.com</loc></url></urlset>', {
status: 200,
})
);
const parser = new SitemapParser({ insecure: false });
await parser.parseFromUrl('https://example.com/sitemap.xml');
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/sitemap.xml',
expect.objectContaining({
dispatcher: undefined,
})
);
});
});
describe('parseSitemap', () => {
it('should parse valid sitemap XML', async () => {
const parser = new SitemapParser();
const xml = `<?xml version="1.0"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://example.com/page1</loc>
<lastmod>2025-01-01</lastmod>
</url>
<url>
<loc>https://example.com/page2</loc>
</url>
</urlset>`;
const result = await parser.parseSitemap(xml);
expect(result.isIndex).toBe(false);
expect(result.urls).toHaveLength(2);
expect(result.urls[0].loc).toBe('https://example.com/page1');
expect(result.urls[0].lastmod).toBe('2025-01-01');
expect(result.urls[1].loc).toBe('https://example.com/page2');
});
it('should handle XML with newlines in declaration', async () => {
const parser = new SitemapParser();
// This is the malformed format from the original issue
const xml = `<?xml
version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="https://example.com/sitemap.xsl" ?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://example.com/</loc>
<lastmod>2025-01-01</lastmod>
</url>
</urlset>`;
const result = await parser.parseSitemap(xml);
expect(result.isIndex).toBe(false);
expect(result.urls).toHaveLength(1);
expect(result.urls[0].loc).toBe('https://example.com/');
});
it('should handle leading whitespace before XML declaration', async () => {
const parser = new SitemapParser();
const xml = `
<?xml version="1.0"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>https://example.com/</loc></url>
</urlset>`;
const result = await parser.parseSitemap(xml);
expect(result.urls).toHaveLength(1);
expect(result.urls[0].loc).toBe('https://example.com/');
});
});
});

89
src/SitemapParser.ts
View file

@ -1,8 +1,50 @@
import { XMLParser } from 'fast-xml-parser';
import { Agent } from 'undici';
import type { SitemapUrl, ParsedSitemap, LoggerFunction } from './types.js';
export interface SitemapParserOptions {
logger?: LoggerFunction;
/** Skip SSL certificate verification (for servers with broken cert chains) */
insecure?: boolean;
}
/**
* Check if an error is related to SSL/TLS certificate verification
* Node.js fetch wraps SSL errors in a TypeError with the actual error in .cause
*/
function isSSLError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const sslErrorPatterns = [
'unable to verify the first certificate',
'certificate has expired',
'self signed certificate',
'CERT_',
'SSL_',
'UNABLE_TO_GET_ISSUER_CERT',
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
'DEPTH_ZERO_SELF_SIGNED_CERT',
];
// Check the error itself
const message = error.message || '';
const code = (error as NodeJS.ErrnoException).code || '';
if (sslErrorPatterns.some(pattern => message.includes(pattern) || code.includes(pattern))) {
return true;
}
// Check error.cause (Node.js fetch wraps SSL errors this way)
const cause = (error as Error & { cause?: Error }).cause;
if (cause instanceof Error) {
const causeMessage = cause.message || '';
const causeCode = (cause as NodeJS.ErrnoException).code || '';
if (sslErrorPatterns.some(pattern => causeMessage.includes(pattern) || causeCode.includes(pattern))) {
return true;
}
}
return false;
}
/**
@ -18,9 +60,11 @@ export interface SitemapParserOptions {
export class SitemapParser {
private xmlParser: XMLParser;
private logger?: LoggerFunction;
private insecure: boolean;
constructor(options: SitemapParserOptions = {}) {
this.logger = options.logger;
this.insecure = options.insecure ?? false;
this.xmlParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
@ -66,27 +110,46 @@ export class SitemapParser {
/**
* Parse a sitemap directly from a URL
*
*
* @param url - The URL of the sitemap to fetch and parse
* @returns Parsed sitemap with URLs and index flag
* @throws Error if fetch fails or XML is invalid
*/
async parseFromUrl(url: string): Promise<ParsedSitemap> {
this.logger?.(`Fetching sitemap from ${url}`, 'info');
const response = await fetch(url, {
headers: {
'User-Agent': 'amb-sitemap-parser/0.1.0',
'Accept': 'application/xml, text/xml, */*',
},
});
if (!response.ok) {
throw new Error(`Failed to fetch sitemap: HTTP ${response.status}`);
// Create custom dispatcher to skip SSL verification if insecure mode is enabled
const dispatcher = this.insecure
? new Agent({ connect: { rejectUnauthorized: false } })
: undefined;
try {
const response = await fetch(url, {
headers: {
'User-Agent': 'amb-sitemap-parser/0.1.0',
'Accept': 'application/xml, text/xml, */*',
},
// @ts-expect-error - dispatcher is a valid undici option for Node.js native fetch
dispatcher,
});
if (!response.ok) {
throw new Error(`Failed to fetch sitemap: HTTP ${response.status}`);
}
const xml = await response.text();
return this.parseSitemap(xml);
} catch (error) {
// Provide helpful error message for SSL certificate errors
if (isSSLError(error)) {
throw new Error(
`SSL certificate verification failed for ${url}. ` +
`The server may have an incomplete certificate chain. ` +
`Use --insecure to bypass certificate verification.`
);
}
throw error;
}
const xml = await response.text();
return this.parseSitemap(xml);
}
/**

96
src/cli.ts
View file

@ -2,9 +2,11 @@
import { Command } from 'commander';
import { SitemapParser } from './SitemapParser.js';
import type { SitemapUrl } from './types.js';
import { PageFetcher } from './PageFetcher.js';
import { MetadataExtractor } from './MetadataExtractor.js';
import { writeJsonLines, validateOutputPath } from './outputWriter.js';
import { Progress } from './progress.js';
const program = new Command();
@ -16,31 +18,38 @@ program
// Parse command
program
.command('parse')
.description('Parse a sitemap and list all URLs')
.argument('<sitemap-url>', 'URL of the sitemap to parse')
.description('Parse one or more sitemaps and list all URLs')
.argument('<sitemap-urls...>', 'One or more sitemap URLs to parse')
.option('-l, --limit <number>', 'Limit to first N URLs', parseInt)
.option('-p, --pretty', 'Pretty-print JSON output', false)
.action(async (sitemapUrl: string, options: { limit?: number; pretty: boolean }) => {
.option('-k, --insecure', 'Skip SSL certificate verification', false)
.action(async (sitemapUrls: string[], options: { limit?: number; pretty: boolean; insecure: boolean }) => {
try {
const parser = new SitemapParser();
const sitemap = await parser.parseFromUrl(sitemapUrl);
const parser = new SitemapParser({ insecure: options.insecure });
let allUrls: string[] = [];
let hasIndex = false;
for (const sitemapUrl of sitemapUrls) {
const sitemap = await parser.parseFromUrl(sitemapUrl);
allUrls.push(...sitemap.urls.map(u => u.loc));
if (sitemap.isIndex) hasIndex = true;
}
let urls = sitemap.urls.map(u => u.loc);
if (options.limit) {
urls = urls.slice(0, options.limit);
allUrls = allUrls.slice(0, options.limit);
}
const output = {
urls,
isIndex: sitemap.isIndex,
count: urls.length,
urls: allUrls,
isIndex: hasIndex,
count: allUrls.length,
};
const json = options.pretty
const json = options.pretty
? JSON.stringify(output, null, 2)
: JSON.stringify(output);
console.log(json);
process.exit(0);
} catch (error) {
@ -52,8 +61,8 @@ program
// Extract command
program
.command('extract')
.description('Full pipeline: parse sitemap, fetch pages, extract educational metadata')
.argument('<sitemap-url>', 'URL of the sitemap to parse')
.description('Full pipeline: parse sitemaps, fetch pages, extract educational metadata')
.argument('<sitemap-urls...>', 'One or more sitemap URLs to parse')
.option('-l, --limit <number>', 'Limit URLs to process', parseInt)
.option('-c, --max-concurrency <number>', 'Maximum concurrent requests', parseInt, 5)
.option('-t, --timeout <number>', 'Request timeout in milliseconds', parseInt, 30000)
@ -61,7 +70,9 @@ program
.option('-p, --pretty', 'Pretty-print JSON output', false)
.option('--jsonl', 'Output resources as newline-delimited JSON to stdout (one resource per line)', false)
.option('-v, --verbose', 'Show progress logs', false)
.action(async (sitemapUrl: string, options: {
.option('-q, --quiet', 'Suppress progress output', false)
.option('-k, --insecure', 'Skip SSL certificate verification', false)
.action(async (sitemapUrls: string[], options: {
limit?: number;
maxConcurrency: number;
timeout: number;
@ -69,6 +80,8 @@ program
pretty: boolean;
jsonl: boolean;
verbose: boolean;
quiet: boolean;
insecure: boolean;
}) => {
try {
// Validate mutually exclusive options
@ -78,6 +91,9 @@ program
process.exit(1);
}
// Progress indicator (quiet when using jsonl to not interfere with output)
const progress = new Progress({ quiet: options.quiet || options.jsonl });
// Logger function that writes to stderr if verbose is enabled
const logger = options.verbose
? (message: string, level: 'info' | 'warn' | 'error') => {
@ -86,24 +102,32 @@ program
}
: undefined;
// Step 1: Parse sitemap
if (options.verbose) {
console.error(`[INFO] Parsing sitemap: ${sitemapUrl}`);
}
const parser = new SitemapParser({ logger });
const sitemap = await parser.parseFromUrl(sitemapUrl);
// Step 1: Parse sitemaps
progress.update('Parsing sitemaps...');
const parser = new SitemapParser({ logger, insecure: options.insecure });
const allSitemapUrls: SitemapUrl[] = [];
let urls = sitemap.urls;
for (const sitemapUrl of sitemapUrls) {
if (options.verbose) {
console.error(`[INFO] Parsing sitemap: ${sitemapUrl}`);
}
const sitemap = await parser.parseFromUrl(sitemapUrl);
allSitemapUrls.push(...sitemap.urls);
}
let urls = allSitemapUrls;
if (options.limit) {
urls = urls.slice(0, options.limit);
}
const totalUrls = urls.length;
progress.done(`Found ${totalUrls} URLs from ${sitemapUrls.length} sitemap(s)`);
if (options.verbose) {
console.error(`[INFO] Found ${totalUrls} URLs to process`);
}
// Step 2: Fetch pages
progress.update(`Fetching pages: 0/${totalUrls}`);
if (options.verbose) {
console.error(`[INFO] Fetching pages with concurrency: ${options.maxConcurrency}`);
}
@ -111,15 +135,21 @@ program
logger,
maxConcurrency: options.maxConcurrency,
timeout: options.timeout,
insecure: options.insecure,
onProgress: (completed, total) => {
progress.update(`Fetching pages: ${completed}/${total}`);
},
});
const fetchResults = await fetcher.fetchPages(urls);
const fetchedCount = fetchResults.filter(r => r.success).length;
progress.done(`Fetched ${fetchedCount}/${totalUrls} pages`);
if (options.verbose) {
console.error(`[INFO] Successfully fetched ${fetchedCount}/${totalUrls} pages`);
}
// Step 3: Extract metadata
progress.update('Extracting metadata...');
if (options.verbose) {
console.error(`[INFO] Extracting metadata from fetched pages`);
}
@ -127,16 +157,19 @@ program
const metadata = await extractor.extractFromPages(fetchResults);
const withMetadataCount = metadata.filter(m => !m.error && (m.jsonLd || m.name || m.description)).length;
progress.done(`Extracted metadata from ${withMetadataCount} pages`);
if (options.verbose) {
console.error(`[INFO] Extracted metadata from ${withMetadataCount}/${fetchedCount} pages`);
}
// Step 4: Filter educational content
progress.update('Filtering educational content...');
if (options.verbose) {
console.error(`[INFO] Filtering for educational content`);
}
const educationalMetadata = MetadataExtractor.filterEducationalMetadata(metadata);
progress.done(`Found ${educationalMetadata.length} educational resources`);
if (options.verbose) {
console.error(`[INFO] Found ${educationalMetadata.length} pages with educational metadata`);
}
@ -230,6 +263,8 @@ program
.option('-p, --pretty', 'Pretty-print JSON output', false)
.option('--jsonl', 'Output resources as newline-delimited JSON to stdout (one resource per line)', false)
.option('-v, --verbose', 'Show progress logs', false)
.option('-q, --quiet', 'Suppress progress output', false)
.option('-k, --insecure', 'Skip SSL certificate verification', false)
.action(async (urls: string[], options: {
maxConcurrency: number;
timeout: number;
@ -237,6 +272,8 @@ program
pretty: boolean;
jsonl: boolean;
verbose: boolean;
quiet: boolean;
insecure: boolean;
}) => {
try {
// Validate mutually exclusive options
@ -246,6 +283,9 @@ program
process.exit(1);
}
// Progress indicator (quiet when using jsonl to not interfere with output)
const progress = new Progress({ quiet: options.quiet || options.jsonl });
// Logger function that writes to stderr if verbose is enabled
const logger = options.verbose
? (message: string, level: 'info' | 'warn' | 'error') => {
@ -271,6 +311,7 @@ program
}
// Step 1: Fetch pages
progress.update(`Fetching pages: 0/${totalUrls}`);
if (options.verbose) {
console.error(`[INFO] Fetching pages with concurrency: ${options.maxConcurrency}`);
}
@ -278,15 +319,21 @@ program
logger,
maxConcurrency: options.maxConcurrency,
timeout: options.timeout,
insecure: options.insecure,
onProgress: (completed, total) => {
progress.update(`Fetching pages: ${completed}/${total}`);
},
});
const fetchResults = await fetcher.fetchPages(sitemapUrls);
const fetchedCount = fetchResults.filter(r => r.success).length;
progress.done(`Fetched ${fetchedCount}/${totalUrls} pages`);
if (options.verbose) {
console.error(`[INFO] Successfully fetched ${fetchedCount}/${totalUrls} pages`);
}
// Step 2: Extract metadata
progress.update('Extracting metadata...');
if (options.verbose) {
console.error(`[INFO] Extracting metadata from fetched pages`);
}
@ -294,16 +341,19 @@ program
const metadata = await extractor.extractFromPages(fetchResults);
const withMetadataCount = metadata.filter(m => !m.error && (m.jsonLd || m.name || m.description)).length;
progress.done(`Extracted metadata from ${withMetadataCount} pages`);
if (options.verbose) {
console.error(`[INFO] Extracted metadata from ${withMetadataCount}/${fetchedCount} pages`);
}
// Step 3: Filter educational content
progress.update('Filtering educational content...');
if (options.verbose) {
console.error(`[INFO] Filtering for educational content`);
}
const educationalMetadata = MetadataExtractor.filterEducationalMetadata(metadata);
progress.done(`Found ${educationalMetadata.length} educational resources`);
if (options.verbose) {
console.error(`[INFO] Found ${educationalMetadata.length} pages with educational metadata`);
}

10
src/outputWriter.ts
View file

@ -34,9 +34,7 @@ export async function validateOutputPath(filepath: string): Promise<void> {
* Writes extracted metadata to a JSON Lines (JSONL) file
*
* Flattens the metadata array so that each JSON-LD object becomes a separate line.
* Each line contains:
* - url: The source URL of the page
* - jsonLd: A single JSON-LD object from that page
* Each line contains a pure JSON-LD object.
*
* This format is ideal for streaming and piping to other tools.
*
@ -66,11 +64,7 @@ export async function writeJsonLines(
}
for (const jsonLdObject of item.jsonLd) {
const record = {
url: item.url,
jsonLd: jsonLdObject,
};
lines.push(JSON.stringify(record));
lines.push(JSON.stringify(jsonLdObject));
}
}

65
src/progress.ts Normal file
View file

@ -0,0 +1,65 @@
/**
* Progress indicator utility for CLI output
* Outputs to stderr to not interfere with JSON output on stdout
*/
export interface ProgressOptions {
/** Suppress all progress output */
quiet?: boolean;
/** Output stream (defaults to stderr) */
stream?: NodeJS.WriteStream;
}
/**
* Simple progress indicator that handles TTY vs non-TTY output
*
* - In TTY mode: Updates progress in place using carriage return
* - In non-TTY mode: Prints each update on a new line
*/
export class Progress {
private quiet: boolean;
private stream: NodeJS.WriteStream;
private isTTY: boolean;
private lastLineLength = 0;
constructor(options: ProgressOptions = {}) {
this.quiet = options.quiet ?? false;
this.stream = options.stream ?? process.stderr;
this.isTTY = this.stream.isTTY ?? false;
}
/**
* Update progress in place (TTY) or print new line (non-TTY)
*/
update(message: string): void {
if (this.quiet) return;
if (this.isTTY) {
// Clear previous line and write new one
const clearLine = ' '.repeat(this.lastLineLength);
this.stream.write(`\r${clearLine}\r${message}`);
this.lastLineLength = message.length;
} else {
this.stream.write(`${message}\n`);
}
}
/**
* Complete current progress line and move to next line
*/
done(message?: string): void {
if (this.quiet) return;
if (message) {
if (this.isTTY) {
const clearLine = ' '.repeat(this.lastLineLength);
this.stream.write(`\r${clearLine}\r${message}\n`);
} else {
this.stream.write(`${message}\n`);
}
} else if (this.isTTY) {
this.stream.write('\n');
}
this.lastLineLength = 0;
}
}

2
src/types.ts
View file

@ -36,6 +36,8 @@ export interface FetchOptions {
maxConcurrency?: number;
userAgent?: string;
delayBetweenRequests?: number;
/** Skip SSL certificate verification (for servers with broken cert chains) */
insecure?: boolean;
}
/**