initial commit
This commit is contained in:
@s.roertgen
commit
4ae0ab146c
13 changed files with 5938 additions and 0 deletions
46
.gitignore
vendored
Normal file
46
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
node_modules
|
||||
|
||||
# Output
|
||||
.output
|
||||
.vercel
|
||||
.netlify
|
||||
.wrangler
|
||||
/.svelte-kit
|
||||
/build
|
||||
dist/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Env
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!.env.test
|
||||
|
||||
# Vite
|
||||
vite.config.js.timestamp-*
|
||||
vite.config.ts.timestamp-*
|
||||
|
||||
# Cline
|
||||
memory-bank
|
||||
.clineignore
|
||||
.clinerules
|
||||
|
||||
data/
|
||||
!tests/data/
|
||||
|
||||
#i18n
|
||||
project.inlang
|
||||
|
||||
# Editor
|
||||
.vscode/
|
||||
.idea/
|
||||
*.sublime-workspace
|
||||
*.sublime-project
|
||||
*.code-workspace
|
||||
|
||||
# Tests
|
||||
coverage/
|
||||
_test/
|
||||
662
README.md
Normal file
662
README.md
Normal file
|
|
@ -0,0 +1,662 @@
|
|||
# amb-sitemap-parser
|
||||
|
||||
A modern TypeScript library for parsing sitemaps and extracting educational metadata from web pages. Built with pure ESM for Node.js 20+.
|
||||
|
||||
## Features
|
||||
|
||||
- 📋 Parse XML sitemaps (regular sitemaps and sitemap indexes)
|
||||
- 🌐 Concurrent page fetching with rate limiting
|
||||
- 🔍 Extract JSON-LD metadata from HTML pages
|
||||
- 📊 Filter educational content automatically
|
||||
- 🎯 Full TypeScript support with exported types
|
||||
- 🚀 Pure ESM for modern JavaScript environments
|
||||
- 🧪 Tested with Vitest
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install amb-sitemap-parser
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
- Node.js >= 20.0.0
|
||||
- ESM-compatible project (set `"type": "module"` in package.json)
|
||||
|
||||
## CLI Usage
|
||||
|
||||
The package includes a command-line interface for parsing sitemaps and extracting educational metadata.
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
# Install globally for CLI usage
|
||||
npm install -g amb-sitemap-parser
|
||||
|
||||
# Or use with npx (no installation needed)
|
||||
npx amb-sitemap-parser --help
|
||||
```
|
||||
|
||||
### Commands
|
||||
|
||||
#### `parse` - Parse a sitemap and list URLs
|
||||
|
||||
```bash
|
||||
# Basic usage
|
||||
amb-sitemap-parser parse https://example.com/sitemap.xml
|
||||
|
||||
# Limit to first 10 URLs
|
||||
amb-sitemap-parser parse https://example.com/sitemap.xml --limit 10
|
||||
|
||||
# Pretty-print JSON output
|
||||
amb-sitemap-parser parse https://example.com/sitemap.xml --pretty
|
||||
|
||||
# Combine options
|
||||
amb-sitemap-parser parse https://example.com/sitemap.xml --limit 20 --pretty
|
||||
```
|
||||
|
||||
**Output:**
|
||||
```json
|
||||
{
|
||||
"urls": ["url1", "url2", "url3"],
|
||||
"isIndex": false,
|
||||
"count": 3
|
||||
}
|
||||
```
|
||||
|
||||
#### `extract` - Extract educational metadata
|
||||
|
||||
Full pipeline: parse sitemap → fetch pages → extract metadata → filter educational content
|
||||
|
||||
```bash
|
||||
# Basic usage
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml
|
||||
|
||||
# With verbose logging (logs to stderr)
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --verbose
|
||||
|
||||
# Limit URLs and adjust concurrency
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --limit 20 --max-concurrency 10
|
||||
|
||||
# Pretty-print output
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --pretty
|
||||
|
||||
# Custom timeout (in milliseconds)
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --timeout 60000
|
||||
|
||||
# Save to file
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml > results.json
|
||||
```
|
||||
|
||||
**Output:**
|
||||
```json
|
||||
{
|
||||
"metadata": [
|
||||
{
|
||||
"url": "https://example.com/course",
|
||||
"title": "Introduction to Python",
|
||||
"description": "Learn Python basics",
|
||||
"jsonLdData": [
|
||||
{
|
||||
"@type": "Course",
|
||||
"name": "Introduction to Python"
|
||||
}
|
||||
],
|
||||
"extractionTime": 45
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"totalUrls": 100,
|
||||
"fetched": 95,
|
||||
"withMetadata": 78,
|
||||
"educational": 42
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### `fetch` - Fetch URL(s) directly and extract metadata
|
||||
|
||||
Extract educational metadata from specific URLs without needing a sitemap.
|
||||
|
||||
```bash
|
||||
# Fetch a single URL
|
||||
amb-sitemap-parser fetch https://example.com/course
|
||||
|
||||
# Fetch multiple URLs
|
||||
amb-sitemap-parser fetch https://example.com/course1 https://example.com/course2
|
||||
|
||||
# With verbose logging
|
||||
amb-sitemap-parser fetch https://example.com/course --verbose
|
||||
|
||||
# Adjust concurrency for multiple URLs
|
||||
amb-sitemap-parser fetch https://example.com/course1 https://example.com/course2 --max-concurrency 10
|
||||
|
||||
# Pretty-print output
|
||||
amb-sitemap-parser fetch https://example.com/course --pretty
|
||||
|
||||
# Custom timeout (in milliseconds)
|
||||
amb-sitemap-parser fetch https://example.com/course --timeout 60000
|
||||
|
||||
# Save to file
|
||||
amb-sitemap-parser fetch https://example.com/course > result.json
|
||||
```
|
||||
|
||||
**Output:** (same format as `extract` command)
|
||||
```json
|
||||
{
|
||||
"metadata": [
|
||||
{
|
||||
"url": "https://example.com/course",
|
||||
"title": "Introduction to Python",
|
||||
"description": "Learn Python basics",
|
||||
"jsonLdData": [
|
||||
{
|
||||
"@type": "Course",
|
||||
"name": "Introduction to Python"
|
||||
}
|
||||
],
|
||||
"extractionTime": 45
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"totalUrls": 1,
|
||||
"fetched": 1,
|
||||
"withMetadata": 1,
|
||||
"educational": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### CLI Options Reference
|
||||
|
||||
#### `parse` command options:
|
||||
- `-l, --limit <number>` - Limit to first N URLs
|
||||
- `-p, --pretty` - Pretty-print JSON output
|
||||
|
||||
#### `extract` command options:
|
||||
- `-l, --limit <number>` - Limit URLs to process
|
||||
- `-c, --max-concurrency <number>` - Maximum concurrent requests (default: 5)
|
||||
- `-t, --timeout <number>` - Request timeout in milliseconds (default: 30000)
|
||||
- `-o, --output <filepath>` - Save metadata to JSONL file (one JSON-LD object per line)
|
||||
- `-p, --pretty` - Pretty-print JSON output
|
||||
- `-v, --verbose` - Show progress logs (written to stderr)
|
||||
|
||||
#### `fetch` command options:
|
||||
- `-c, --max-concurrency <number>` - Maximum concurrent requests (default: 5)
|
||||
- `-t, --timeout <number>` - Request timeout in milliseconds (default: 30000)
|
||||
- `-o, --output <filepath>` - Save metadata to JSONL file (one JSON-LD object per line)
|
||||
- `-p, --pretty` - Pretty-print JSON output
|
||||
- `-v, --verbose` - Show progress logs (written to stderr)
|
||||
|
||||
### JSONL Output Format
|
||||
|
||||
When using the `--output` option, metadata is saved in **JSON Lines (JSONL)** format - one JSON-LD object per line. This format is ideal for streaming, piping, and processing with standard Unix tools.
|
||||
|
||||
#### How It Works
|
||||
|
||||
Each JSON-LD object from a page becomes a separate line in the file:
|
||||
|
||||
```jsonl
|
||||
{"url":"https://example.com/course1","jsonLd":{"@type":"Course","name":"Python 101"}}
|
||||
{"url":"https://example.com/course1","jsonLd":{"@type":"LearningResource","name":"Exercise 1"}}
|
||||
{"url":"https://example.com/course2","jsonLd":{"@type":"Course","name":"JavaScript Basics"}}
|
||||
```
|
||||
|
||||
**Key benefits:**
|
||||
- ✅ Each line is a complete, independent record
|
||||
- ✅ Stream-friendly (process one record at a time)
|
||||
- ✅ Perfect for Unix pipelines (`jq`, `grep`, `awk`)
|
||||
- ✅ Easy to parallelize processing
|
||||
- ✅ If a page has multiple JSON-LD objects, each gets its own line
|
||||
|
||||
#### Output Behavior
|
||||
|
||||
**With `--output` flag:**
|
||||
```bash
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --output results.jsonl
|
||||
```
|
||||
|
||||
- **File (results.jsonl)**: Contains JSONL records (one JSON-LD object per line)
|
||||
- **stdout**: Summary statistics only
|
||||
```json
|
||||
{
|
||||
"summary": {
|
||||
"totalUrls": 100,
|
||||
"fetched": 95,
|
||||
"withMetadata": 78,
|
||||
"educational": 45,
|
||||
"recordsWritten": 67
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Without `--output` flag:**
|
||||
```bash
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml
|
||||
```
|
||||
|
||||
- **stdout**: Full metadata array + summary (existing behavior)
|
||||
|
||||
### CLI Examples
|
||||
|
||||
```bash
|
||||
# Quick test: parse and see first 5 URLs
|
||||
amb-sitemap-parser parse https://example.com/sitemap.xml --limit 5 --pretty
|
||||
|
||||
# Extract metadata from first 10 URLs with verbose logging
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --limit 10 --verbose --pretty
|
||||
|
||||
# High-performance extraction: 20 concurrent requests
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --max-concurrency 20
|
||||
|
||||
# Save to JSONL file
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --output results.jsonl --verbose
|
||||
|
||||
# Save from direct URL fetch
|
||||
amb-sitemap-parser fetch https://example.com/course --output course.jsonl
|
||||
|
||||
# Pipeline with jq for further processing
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml | jq '.metadata[] | select(.jsonLd != null)'
|
||||
```
|
||||
|
||||
### JSONL Piping Examples
|
||||
|
||||
The JSONL format enables powerful command-line workflows:
|
||||
|
||||
```bash
|
||||
# Save metadata to file
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml -o results.jsonl -v
|
||||
|
||||
# Count total records
|
||||
wc -l < results.jsonl
|
||||
|
||||
# Filter by type: only Courses
|
||||
cat results.jsonl | jq 'select(.jsonLd."@type" == "Course")'
|
||||
|
||||
# Extract all course names
|
||||
cat results.jsonl | jq -r '.jsonLd.name' | sort | uniq
|
||||
|
||||
# Count by type
|
||||
cat results.jsonl | jq -r '.jsonLd."@type"' | sort | uniq -c
|
||||
|
||||
# Find courses from specific domain
|
||||
cat results.jsonl | jq 'select(.url | contains("example.com"))'
|
||||
|
||||
# Get URLs with specific property
|
||||
cat results.jsonl | jq -r 'select(.jsonLd.educationalLevel) | .url'
|
||||
|
||||
# Process one record at a time
|
||||
cat results.jsonl | while read line; do
|
||||
echo "$line" | jq '.jsonLd.name'
|
||||
done
|
||||
|
||||
# Parallel processing with GNU parallel
|
||||
cat results.jsonl | parallel --pipe -L1 'echo {} | jq ".jsonLd.name"'
|
||||
|
||||
# Convert to CSV (name and URL only)
|
||||
cat results.jsonl | jq -r '[.jsonLd.name, .url] | @csv'
|
||||
|
||||
# Filter and save to new file
|
||||
cat results.jsonl | jq 'select(.jsonLd."@type" == "Course")' > courses-only.jsonl
|
||||
```
|
||||
|
||||
## Integration with AMB-Nostr-Converter
|
||||
|
||||
Convert extracted AMB resources directly to Nostr events using the `--jsonl` flag for seamless streaming between tools.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Install the AMB-Nostr converter:
|
||||
|
||||
```bash
|
||||
npm install -g @edufeed-org/amb-nostr-converter
|
||||
# Or use npx for on-demand usage
|
||||
```
|
||||
|
||||
### Basic Pipeline
|
||||
|
||||
The `--jsonl` flag outputs one AMB resource per line to stdout, perfect for piping:
|
||||
|
||||
```bash
|
||||
# Extract and convert a single resource
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl --limit 1 | \
|
||||
npx amb-convert amb:nostr --pretty
|
||||
|
||||
# Extract from direct URL
|
||||
amb-sitemap-parser fetch https://example.com/course --jsonl | \
|
||||
npx amb-convert amb:nostr --pretty
|
||||
```
|
||||
|
||||
### Processing Multiple Resources
|
||||
|
||||
Convert all resources from a sitemap to Nostr events:
|
||||
|
||||
```bash
|
||||
# Using a while loop
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr
|
||||
done > nostr-events.jsonl
|
||||
|
||||
# With verbose logging (logs go to stderr, data to stdout)
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl -v | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr
|
||||
done > nostr-events.jsonl
|
||||
```
|
||||
|
||||
### Signing Nostr Events
|
||||
|
||||
Sign events with your Nostr private key:
|
||||
|
||||
```bash
|
||||
# Using nsec (bech32 format)
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC
|
||||
done > signed-events.jsonl
|
||||
|
||||
# Using hex private key
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr --private-key $NOSTR_PRIVATE_KEY
|
||||
done > signed-events.jsonl
|
||||
```
|
||||
|
||||
### Complete Workflow Examples
|
||||
|
||||
```bash
|
||||
# 1. Test with a single URL first
|
||||
amb-sitemap-parser fetch https://example.com/course --jsonl -v | \
|
||||
npx amb-convert amb:nostr --pretty
|
||||
|
||||
# 2. Process first 10 resources from sitemap
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl -l 10 -v | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC
|
||||
done > events.jsonl
|
||||
|
||||
# 3. Full sitemap conversion with signed events
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl --max-concurrency 10 -v | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC --pretty
|
||||
done > all-events.jsonl 2> conversion.log
|
||||
|
||||
# 4. Filter and convert only Course types
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml --jsonl | \
|
||||
jq -c 'select(.type | contains(["Course"]))' | \
|
||||
while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr
|
||||
done > courses-events.jsonl
|
||||
|
||||
# 5. Save AMB resources and convert separately
|
||||
amb-sitemap-parser extract https://example.com/sitemap.xml -o amb-resources.jsonl -v
|
||||
cat amb-resources.jsonl | while read resource; do
|
||||
echo "$resource" | npx amb-convert amb:nostr --nsec $NOSTR_NSEC
|
||||
done > nostr-events.jsonl
|
||||
```
|
||||
|
||||
### Why `--jsonl` Flag?
|
||||
|
||||
The `--jsonl` flag is specifically designed for tool integration:
|
||||
|
||||
**Without `--jsonl` (default):**
|
||||
```json
|
||||
{
|
||||
"metadata": [
|
||||
{ "url": "...", "jsonLd": [{...}] },
|
||||
{ "url": "...", "jsonLd": [{...}] }
|
||||
],
|
||||
"summary": {...}
|
||||
}
|
||||
```
|
||||
❌ Complex nested structure, hard to pipe
|
||||
|
||||
**With `--jsonl`:**
|
||||
```
|
||||
{AMB resource 1}
|
||||
{AMB resource 2}
|
||||
{AMB resource 3}
|
||||
```
|
||||
✅ One resource per line, perfect for streaming
|
||||
|
||||
### Integration Benefits
|
||||
|
||||
- ✅ **Streaming**: Process resources one at a time, low memory usage
|
||||
- ✅ **Composable**: Combine with standard Unix tools (`jq`, `grep`, `awk`)
|
||||
- ✅ **Resumable**: Stop and resume processing at any point
|
||||
- ✅ **Flexible**: Filter, transform, and route data as needed
|
||||
- ✅ **Observable**: Use verbose mode (`-v`) to monitor progress via stderr
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
**Issue**: "Cannot use --output and --jsonl together"
|
||||
```bash
|
||||
# ❌ Wrong
|
||||
amb-sitemap-parser extract url --output file.jsonl --jsonl
|
||||
|
||||
# ✅ Correct (save to file)
|
||||
amb-sitemap-parser extract url --output file.jsonl
|
||||
|
||||
# ✅ Correct (pipe to stdout)
|
||||
amb-sitemap-parser extract url --jsonl > file.jsonl
|
||||
```
|
||||
|
||||
**Issue**: No output when piping
|
||||
```bash
|
||||
# Make sure to use --jsonl flag for streaming
|
||||
amb-sitemap-parser extract url --jsonl | npx amb-convert amb:nostr
|
||||
```
|
||||
|
||||
**Issue**: Need to see progress while piping
|
||||
```bash
|
||||
# Use --verbose flag (logs go to stderr, data to stdout)
|
||||
amb-sitemap-parser extract url --jsonl -v | npx amb-convert amb:nostr 2>&1
|
||||
```
|
||||
|
||||
## Programmatic Usage (Node.js)
|
||||
|
||||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { SitemapParser, PageFetcher, MetadataExtractor } from 'amb-sitemap-parser';
|
||||
|
||||
// Parse a sitemap
|
||||
const parser = new SitemapParser();
|
||||
const sitemap = await parser.parseFromUrl('https://example.com/sitemap.xml');
|
||||
|
||||
// Fetch pages
|
||||
const fetcher = new PageFetcher({ maxConcurrency: 5 });
|
||||
const results = await fetcher.fetchPages(sitemap.urls.slice(0, 10));
|
||||
|
||||
// Extract metadata
|
||||
const extractor = new MetadataExtractor();
|
||||
const metadata = await extractor.extractFromPages(results);
|
||||
|
||||
// Filter educational content
|
||||
const educational = MetadataExtractor.filterEducationalMetadata(metadata);
|
||||
console.log(educational);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### SitemapParser
|
||||
|
||||
Parse XML sitemaps and extract URLs.
|
||||
|
||||
```typescript
|
||||
const parser = new SitemapParser({
|
||||
logger: (msg, level) => console.log(`[${level}] ${msg}`)
|
||||
});
|
||||
|
||||
// Parse from string
|
||||
const sitemap = await parser.parseSitemap(xmlContent);
|
||||
|
||||
// Parse from URL
|
||||
const sitemap = await parser.parseFromUrl('https://example.com/sitemap.xml');
|
||||
|
||||
// Validate URL
|
||||
const isValid = SitemapParser.isValidSitemapUrl(url);
|
||||
|
||||
// Filter educational URLs
|
||||
const filtered = SitemapParser.filterEducationalUrls(sitemap.urls);
|
||||
```
|
||||
|
||||
### PageFetcher
|
||||
|
||||
Fetch web pages with concurrency control and rate limiting.
|
||||
|
||||
```typescript
|
||||
const fetcher = new PageFetcher({
|
||||
maxConcurrency: 5,
|
||||
timeout: 30000,
|
||||
delayBetweenRequests: 100,
|
||||
retryAttempts: 2,
|
||||
retryDelay: 1000,
|
||||
logger: (msg, level) => console.log(`[${level}] ${msg}`)
|
||||
});
|
||||
|
||||
// Fetch multiple pages
|
||||
const results = await fetcher.fetchPages(urls);
|
||||
|
||||
// Fetch single page
|
||||
const result = await fetcher.fetchSinglePage('https://example.com/page');
|
||||
|
||||
// Validate URL
|
||||
const isValid = PageFetcher.isValidUrl(url);
|
||||
|
||||
// Filter valid URLs
|
||||
const validUrls = PageFetcher.filterValidUrls(urls);
|
||||
```
|
||||
|
||||
### MetadataExtractor
|
||||
|
||||
Extract metadata including JSON-LD from HTML pages.
|
||||
|
||||
```typescript
|
||||
const extractor = new MetadataExtractor({
|
||||
validateSchema: false,
|
||||
logger: (msg, level) => console.log(`[${level}] ${msg}`)
|
||||
});
|
||||
|
||||
// Extract from multiple pages
|
||||
const metadata = await extractor.extractFromPages(fetchResults);
|
||||
|
||||
// Extract from single page
|
||||
const metadata = await extractor.extractFromPage(fetchResult);
|
||||
|
||||
// Filter educational metadata
|
||||
const educational = MetadataExtractor.filterEducationalMetadata(metadata);
|
||||
|
||||
// Check if has valid content
|
||||
const hasContent = MetadataExtractor.hasValidContent(metadata);
|
||||
```
|
||||
|
||||
## Types
|
||||
|
||||
All types are exported and can be imported:
|
||||
|
||||
```typescript
|
||||
import type {
|
||||
SitemapUrl,
|
||||
ParsedSitemap,
|
||||
FetchResult,
|
||||
FetchOptions,
|
||||
ExtractedMetadata,
|
||||
LoggerFunction,
|
||||
} from 'amb-sitemap-parser';
|
||||
```
|
||||
|
||||
## Tree-Shakeable Imports
|
||||
|
||||
Import only what you need for smaller bundle sizes:
|
||||
|
||||
```typescript
|
||||
import { SitemapParser } from 'amb-sitemap-parser/sitemap';
|
||||
import { PageFetcher } from 'amb-sitemap-parser/fetcher';
|
||||
import { MetadataExtractor } from 'amb-sitemap-parser/extractor';
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Complete Workflow
|
||||
|
||||
```typescript
|
||||
import { SitemapParser, PageFetcher, MetadataExtractor } from 'amb-sitemap-parser';
|
||||
|
||||
async function processSitemap(sitemapUrl: string) {
|
||||
// Initialize components
|
||||
const parser = new SitemapParser();
|
||||
const fetcher = new PageFetcher({ maxConcurrency: 5 });
|
||||
const extractor = new MetadataExtractor();
|
||||
|
||||
// Parse sitemap
|
||||
const sitemap = await parser.parseFromUrl(sitemapUrl);
|
||||
console.log(`Found ${sitemap.urls.length} URLs`);
|
||||
|
||||
// Limit to first 50 URLs
|
||||
const urlsToProcess = sitemap.urls.slice(0, 50);
|
||||
|
||||
// Fetch pages
|
||||
const results = await fetcher.fetchPages(urlsToProcess);
|
||||
console.log(`Fetched ${results.filter(r => r.success).length} pages successfully`);
|
||||
|
||||
// Extract metadata
|
||||
const metadata = await extractor.extractFromPages(results);
|
||||
|
||||
// Filter educational content
|
||||
const educational = MetadataExtractor.filterEducationalMetadata(metadata);
|
||||
console.log(`Found ${educational.length} educational resources`);
|
||||
|
||||
return educational;
|
||||
}
|
||||
```
|
||||
|
||||
### With Custom Logger
|
||||
|
||||
```typescript
|
||||
const logger = (message: string, level: 'info' | 'warn' | 'error') => {
|
||||
const timestamp = new Date().toISOString();
|
||||
console.log(`[${timestamp}] [${level.toUpperCase()}] ${message}`);
|
||||
};
|
||||
|
||||
const parser = new SitemapParser({ logger });
|
||||
const fetcher = new PageFetcher({ logger, maxConcurrency: 3 });
|
||||
const extractor = new MetadataExtractor({ logger });
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
npm install
|
||||
|
||||
# Build the library
|
||||
npm run build
|
||||
|
||||
# Run tests
|
||||
npm test
|
||||
|
||||
# Run tests in watch mode
|
||||
npm run test:watch
|
||||
|
||||
# Run tests with UI
|
||||
npm run test:ui
|
||||
|
||||
# Generate coverage report
|
||||
npm run test:coverage
|
||||
|
||||
# Lint code
|
||||
npm run lint
|
||||
|
||||
# Format code
|
||||
npm run format
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
## Contributing
|
||||
|
||||
Contributions are welcome! Please feel free to submit a Pull Request.
|
||||
3918
package-lock.json
generated
Normal file
3918
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
79
package.json
Normal file
79
package.json
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
{
|
||||
"name": "amb-sitemap-parser",
|
||||
"version": "0.1.0",
|
||||
"description": "Parse sitemaps and extract educational metadata from web pages",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"bin": {
|
||||
"amb-sitemap-parser": "./dist/cli.js"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
},
|
||||
"./sitemap": {
|
||||
"types": "./dist/SitemapParser.d.ts",
|
||||
"import": "./dist/SitemapParser.js"
|
||||
},
|
||||
"./fetcher": {
|
||||
"types": "./dist/PageFetcher.d.ts",
|
||||
"import": "./dist/PageFetcher.js"
|
||||
},
|
||||
"./extractor": {
|
||||
"types": "./dist/MetadataExtractor.d.ts",
|
||||
"import": "./dist/MetadataExtractor.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"cli": "node dist/cli.js",
|
||||
"dev": "tsc --watch",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:ui": "vitest --ui",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "eslint src/**/*.ts",
|
||||
"lint:fix": "eslint src/**/*.ts --fix",
|
||||
"format": "prettier --write \"src/**/*.ts\"",
|
||||
"prepublishOnly": "npm run test && npm run build"
|
||||
},
|
||||
"keywords": [
|
||||
"sitemap",
|
||||
"parser",
|
||||
"metadata",
|
||||
"json-ld",
|
||||
"crawler",
|
||||
"educational",
|
||||
"oer",
|
||||
"amb",
|
||||
"esm"
|
||||
],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.1.2",
|
||||
"commander": "^14.0.2",
|
||||
"fast-xml-parser": "^5.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.46.4",
|
||||
"@typescript-eslint/parser": "^8.46.4",
|
||||
"@vitest/coverage-v8": "^2.1.8",
|
||||
"@vitest/ui": "^2.1.8",
|
||||
"eslint": "^9.39.1",
|
||||
"prettier": "^3.6.2",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^2.1.8"
|
||||
}
|
||||
}
|
||||
238
src/MetadataExtractor.ts
Normal file
238
src/MetadataExtractor.ts
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import * as cheerio from 'cheerio';
|
||||
import type { FetchResult, ExtractedMetadata, LoggerFunction } from './types.js';
|
||||
|
||||
export interface MetadataExtractorOptions {
|
||||
logger?: LoggerFunction;
|
||||
validateSchema?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts metadata (especially JSON-LD) from HTML pages
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const extractor = new MetadataExtractor();
|
||||
* const metadata = await extractor.extractFromPage(fetchResult);
|
||||
* ```
|
||||
*/
|
||||
export class MetadataExtractor {
|
||||
private logger?: LoggerFunction;
|
||||
private validateSchema: boolean;
|
||||
|
||||
constructor(options: MetadataExtractorOptions = {}) {
|
||||
this.logger = options.logger;
|
||||
this.validateSchema = options.validateSchema ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract metadata from multiple fetch results
|
||||
*
|
||||
* @param results - Array of fetch results
|
||||
* @returns Array of extracted metadata
|
||||
*/
|
||||
async extractFromPages(results: FetchResult[]): Promise<ExtractedMetadata[]> {
|
||||
const extractions: ExtractedMetadata[] = [];
|
||||
this.logger?.(`Starting metadata extraction from ${results.length} pages...`, 'info');
|
||||
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const result = results[i];
|
||||
if (!result) continue; // Skip if somehow undefined
|
||||
|
||||
this.logger?.(`Processing page ${i + 1}/${results.length}: ${result.url}`, 'info');
|
||||
|
||||
try {
|
||||
const extraction = await this.extractFromPage(result);
|
||||
extractions.push(extraction);
|
||||
|
||||
if (extraction.jsonLd && extraction.jsonLd.length > 0) {
|
||||
this.logger?.(`Found ${extraction.jsonLd.length} JSON-LD objects in ${result.url}`, 'info');
|
||||
} else {
|
||||
this.logger?.(`No JSON-LD data found in ${result.url}`, 'warn');
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger?.(`Failed to extract metadata from ${result.url}: ${error}`, 'error');
|
||||
extractions.push({
|
||||
url: result.url,
|
||||
extractionTime: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown extraction error',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const successfulExtractions = extractions.filter(e => !e.error && (e.jsonLd || e.name || e.description));
|
||||
this.logger?.(`Metadata extraction complete: ${successfulExtractions.length}/${extractions.length} pages had extractable content`, 'info');
|
||||
|
||||
return extractions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract metadata from a single page result
|
||||
*
|
||||
* @param result - Fetch result containing HTML
|
||||
* @returns Extracted metadata including JSON-LD, name, and description
|
||||
*/
|
||||
async extractFromPage(result: FetchResult): Promise<ExtractedMetadata> {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
if (!result.success || !result.html) {
|
||||
return {
|
||||
url: result.url,
|
||||
extractionTime: Date.now() - startTime,
|
||||
error: result.error || 'No HTML content available',
|
||||
};
|
||||
}
|
||||
|
||||
const $ = cheerio.load(result.html);
|
||||
|
||||
// Extract JSON-LD data
|
||||
const jsonLdData = this.extractJsonLdData($);
|
||||
|
||||
// Extract basic metadata
|
||||
const title = this.extractTitle($);
|
||||
const description = this.extractDescription($);
|
||||
|
||||
const metadata: ExtractedMetadata = {
|
||||
url: result.url,
|
||||
extractionTime: Date.now() - startTime,
|
||||
};
|
||||
|
||||
if (jsonLdData.length > 0) {
|
||||
metadata.jsonLd = jsonLdData;
|
||||
}
|
||||
|
||||
if (title) {
|
||||
metadata.name = title;
|
||||
}
|
||||
|
||||
if (description) {
|
||||
metadata.description = description;
|
||||
}
|
||||
|
||||
return metadata;
|
||||
|
||||
} catch (error) {
|
||||
return {
|
||||
url: result.url,
|
||||
extractionTime: Date.now() - startTime,
|
||||
error: error instanceof Error ? error.message : 'Unknown extraction error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract JSON-LD script tags from HTML
|
||||
*/
|
||||
private extractJsonLdData($: cheerio.CheerioAPI): any[] {
|
||||
const jsonLdScripts: any[] = [];
|
||||
|
||||
$('script[type="application/ld+json"]').each((_: number, element: any) => {
|
||||
try {
|
||||
// Use cheerio's text() method which automatically decodes HTML entities
|
||||
const scriptContent = $(element).text().trim();
|
||||
|
||||
if (scriptContent) {
|
||||
const parsed = JSON.parse(scriptContent);
|
||||
jsonLdScripts.push(parsed);
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip malformed JSON-LD
|
||||
this.logger?.(
|
||||
`Skipping malformed JSON-LD in ${$(element).closest('html').find('title').text() || 'unknown page'}: ${error}`,
|
||||
'warn'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return jsonLdScripts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract page title
|
||||
*/
|
||||
private extractTitle($: cheerio.CheerioAPI): string | undefined {
|
||||
// Try different title sources in order of preference
|
||||
const titleSelectors = [
|
||||
'title',
|
||||
'meta[property="og:title"]',
|
||||
'meta[name="twitter:title"]',
|
||||
'h1',
|
||||
];
|
||||
|
||||
for (const selector of titleSelectors) {
|
||||
if (selector === 'title') {
|
||||
const title = $('title').text().trim();
|
||||
if (title) return title;
|
||||
} else if (selector.startsWith('meta[')) {
|
||||
const content = $(selector).attr('content')?.trim();
|
||||
if (content) return content;
|
||||
} else {
|
||||
const text = $(selector).first().text().trim();
|
||||
if (text) return text;
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract page description
|
||||
*/
|
||||
private extractDescription($: cheerio.CheerioAPI): string | undefined {
|
||||
// Try different description sources in order of preference
|
||||
const descSelectors = [
|
||||
'meta[name="description"]',
|
||||
'meta[property="og:description"]',
|
||||
'meta[name="twitter:description"]',
|
||||
];
|
||||
|
||||
for (const selector of descSelectors) {
|
||||
const content = $(selector).attr('content')?.trim();
|
||||
if (content) return content;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter metadata to only include AMB-relevant data
|
||||
* This looks for LearningResource, Course, or educational schema.org types
|
||||
*
|
||||
* @param metadata - Array of extracted metadata
|
||||
* @returns Filtered array containing only educational content
|
||||
*/
|
||||
static filterEducationalMetadata(metadata: ExtractedMetadata[]): ExtractedMetadata[] {
|
||||
return metadata.filter(item => {
|
||||
if (!item.jsonLd) return false;
|
||||
|
||||
return item.jsonLd.some((data: any) => {
|
||||
const type = data['@type'] || data.type;
|
||||
if (!type) return false;
|
||||
|
||||
const types = Array.isArray(type) ? type : [type];
|
||||
|
||||
return types.some((t: string) =>
|
||||
t.includes('LearningResource') ||
|
||||
t.includes('Course') ||
|
||||
t.includes('Educational') ||
|
||||
t.includes('CreativeWork') ||
|
||||
t.includes('Article')
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that extracted data has meaningful content
|
||||
*
|
||||
* @param metadata - Extracted metadata to validate
|
||||
* @returns True if the metadata has valid content
|
||||
*/
|
||||
static hasValidContent(metadata: ExtractedMetadata): boolean {
|
||||
return !!(
|
||||
metadata.jsonLd ||
|
||||
metadata.name ||
|
||||
metadata.description
|
||||
);
|
||||
}
|
||||
}
|
||||
225
src/PageFetcher.ts
Normal file
225
src/PageFetcher.ts
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
import type { SitemapUrl, FetchResult, FetchOptions, LoggerFunction } from './types.js';
|
||||
|
||||
export interface PageFetcherOptions extends FetchOptions {
|
||||
logger?: LoggerFunction;
|
||||
retryAttempts?: number;
|
||||
retryDelay?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches web pages with concurrency control and rate limiting
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const fetcher = new PageFetcher({ maxConcurrency: 5 });
|
||||
* const results = await fetcher.fetchPages(urls);
|
||||
* ```
|
||||
*/
|
||||
export class PageFetcher {
|
||||
private defaultOptions: Required<Omit<PageFetcherOptions, 'logger' | 'retryAttempts' | 'retryDelay'>>;
|
||||
private logger?: LoggerFunction;
|
||||
private retryAttempts: number;
|
||||
private retryDelay: number;
|
||||
|
||||
constructor(options: PageFetcherOptions = {}) {
|
||||
this.logger = options.logger;
|
||||
this.retryAttempts = options.retryAttempts ?? 0;
|
||||
this.retryDelay = options.retryDelay ?? 1000;
|
||||
this.defaultOptions = {
|
||||
timeout: options.timeout ?? 30000,
|
||||
maxConcurrency: options.maxConcurrency ?? 5,
|
||||
userAgent: options.userAgent ?? 'amb-sitemap-parser/0.1.0 (Educational Research)',
|
||||
delayBetweenRequests: options.delayBetweenRequests ?? 100,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch multiple pages concurrently with rate limiting
|
||||
*
|
||||
* @param urls - Array of sitemap URLs to fetch
|
||||
* @param options - Override default fetch options
|
||||
* @returns Array of fetch results
|
||||
*/
|
||||
async fetchPages(urls: SitemapUrl[], options: Partial<FetchOptions> = {}): Promise<FetchResult[]> {
|
||||
const opts = { ...this.defaultOptions, ...options };
|
||||
const results: FetchResult[] = [];
|
||||
const totalBatches = Math.ceil(urls.length / opts.maxConcurrency);
|
||||
|
||||
this.logger?.(`Starting to fetch ${urls.length} pages in ${totalBatches} batches (max ${opts.maxConcurrency} concurrent)`, 'info');
|
||||
|
||||
// Process URLs in batches to control concurrency
|
||||
for (let i = 0; i < urls.length; i += opts.maxConcurrency) {
|
||||
const batchIndex = Math.floor(i / opts.maxConcurrency) + 1;
|
||||
const batch = urls.slice(i, i + opts.maxConcurrency);
|
||||
this.logger?.(`Processing batch ${batchIndex}/${totalBatches} (${batch.length} URLs)`, 'info');
|
||||
|
||||
const batchPromises = batch.map(url => this.fetchSinglePage(url.loc, opts));
|
||||
|
||||
try {
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
results.push(...batchResults);
|
||||
|
||||
const successfulInBatch = batchResults.filter(r => r.success).length;
|
||||
this.logger?.(`Batch ${batchIndex} complete: ${successfulInBatch}/${batch.length} successful`, 'info');
|
||||
} catch (error) {
|
||||
this.logger?.(`Batch ${batchIndex} failed: ${error}`, 'error');
|
||||
// Add failed results for all URLs in this batch
|
||||
const failedResults = batch.map(url => ({
|
||||
url: url.loc,
|
||||
success: false,
|
||||
error: 'Batch processing failed',
|
||||
}));
|
||||
results.push(...failedResults);
|
||||
}
|
||||
|
||||
// Add delay between batches if not the last batch
|
||||
if (i + opts.maxConcurrency < urls.length) {
|
||||
await this.delay(opts.delayBetweenRequests * opts.maxConcurrency);
|
||||
}
|
||||
}
|
||||
|
||||
const totalSuccessful = results.filter(r => r.success).length;
|
||||
this.logger?.(`Page fetching complete: ${totalSuccessful}/${results.length} pages fetched successfully`, 'info');
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a single page with timeout and error handling
|
||||
*
|
||||
* @param url - The URL to fetch
|
||||
* @param options - Fetch options
|
||||
* @returns Fetch result with HTML content or error
|
||||
*/
|
||||
async fetchSinglePage(url: string, options?: Required<Omit<FetchOptions, 'maxConcurrency' | 'delayBetweenRequests'>>): Promise<FetchResult> {
|
||||
const opts = options ?? {
|
||||
timeout: this.defaultOptions.timeout,
|
||||
userAgent: this.defaultOptions.userAgent,
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
for (let attempt = 0; attempt <= this.retryAttempts; attempt++) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), opts.timeout);
|
||||
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'User-Agent': opts.userAgent,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
},
|
||||
redirect: 'follow',
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
if (!response.ok) {
|
||||
if (attempt < this.retryAttempts) {
|
||||
this.logger?.(`Retry attempt ${attempt + 1}/${this.retryAttempts} for ${url}`, 'warn');
|
||||
await this.delay(this.retryDelay * (attempt + 1));
|
||||
continue;
|
||||
}
|
||||
|
||||
return {
|
||||
url,
|
||||
success: false,
|
||||
error: `HTTP ${response.status}: ${response.statusText}`,
|
||||
statusCode: response.status,
|
||||
responseTime,
|
||||
};
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
|
||||
return {
|
||||
url,
|
||||
success: true,
|
||||
html,
|
||||
statusCode: response.status,
|
||||
responseTime,
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
if (attempt < this.retryAttempts) {
|
||||
this.logger?.(`Retry attempt ${attempt + 1}/${this.retryAttempts} for ${url}`, 'warn');
|
||||
await this.delay(this.retryDelay * (attempt + 1));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (error instanceof Error) {
|
||||
if (error.name === 'AbortError') {
|
||||
return {
|
||||
url,
|
||||
success: false,
|
||||
error: 'Request timeout',
|
||||
responseTime,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
url,
|
||||
success: false,
|
||||
error: error.message,
|
||||
responseTime,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
url,
|
||||
success: false,
|
||||
error: 'Unknown error occurred',
|
||||
responseTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// This should never be reached due to the loop structure, but TypeScript needs it
|
||||
return {
|
||||
url,
|
||||
success: false,
|
||||
error: 'Max retries exceeded',
|
||||
responseTime: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method to add delay between requests
|
||||
*/
|
||||
private delay(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate URL format
|
||||
*
|
||||
* @param url - The URL to validate
|
||||
* @returns True if the URL is valid
|
||||
*/
|
||||
static isValidUrl(url: string): boolean {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
return urlObj.protocol === 'http:' || urlObj.protocol === 'https:';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter out invalid URLs before fetching
|
||||
*
|
||||
* @param urls - Array of sitemap URLs to filter
|
||||
* @returns Filtered array of valid URLs
|
||||
*/
|
||||
static filterValidUrls(urls: SitemapUrl[]): SitemapUrl[] {
|
||||
return urls.filter(url => this.isValidUrl(url.loc));
|
||||
}
|
||||
}
|
||||
166
src/SitemapParser.ts
Normal file
166
src/SitemapParser.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import { XMLParser } from 'fast-xml-parser';
|
||||
import type { SitemapUrl, ParsedSitemap, LoggerFunction } from './types.js';
|
||||
|
||||
export interface SitemapParserOptions {
|
||||
logger?: LoggerFunction;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parser for XML sitemap files
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const parser = new SitemapParser();
|
||||
* const sitemap = await parser.parseSitemap(xmlContent);
|
||||
* console.log(sitemap.urls);
|
||||
* ```
|
||||
*/
|
||||
export class SitemapParser {
|
||||
private xmlParser: XMLParser;
|
||||
private logger?: LoggerFunction;
|
||||
|
||||
constructor(options: SitemapParserOptions = {}) {
|
||||
this.logger = options.logger;
|
||||
this.xmlParser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
allowBooleanAttributes: true,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a sitemap XML string and extract URLs
|
||||
*
|
||||
* @param xmlContent - The XML content to parse
|
||||
* @returns Parsed sitemap with URLs and index flag
|
||||
* @throws Error if XML is malformed or invalid
|
||||
*/
|
||||
async parseSitemap(xmlContent: string): Promise<ParsedSitemap> {
|
||||
try {
|
||||
// Trim leading whitespace to handle malformed sitemaps with empty lines before XML declaration
|
||||
const trimmedContent = xmlContent.trimStart();
|
||||
const parsed = this.xmlParser.parse(trimmedContent);
|
||||
|
||||
// Check if this is a sitemap index
|
||||
if (parsed.sitemapindex) {
|
||||
return {
|
||||
urls: this.extractUrlsFromIndex(parsed.sitemapindex),
|
||||
isIndex: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Regular sitemap
|
||||
if (parsed.urlset) {
|
||||
return {
|
||||
urls: this.extractUrlsFromUrlset(parsed.urlset),
|
||||
isIndex: false,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error('Invalid sitemap format: missing urlset or sitemapindex');
|
||||
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to parse sitemap XML: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a sitemap directly from a URL
|
||||
*
|
||||
* @param url - The URL of the sitemap to fetch and parse
|
||||
* @returns Parsed sitemap with URLs and index flag
|
||||
* @throws Error if fetch fails or XML is invalid
|
||||
*/
|
||||
async parseFromUrl(url: string): Promise<ParsedSitemap> {
|
||||
this.logger?.(`Fetching sitemap from ${url}`, 'info');
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'amb-sitemap-parser/0.1.0',
|
||||
'Accept': 'application/xml, text/xml, */*',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch sitemap: HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const xml = await response.text();
|
||||
return this.parseSitemap(xml);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract URLs from a sitemap index
|
||||
*/
|
||||
private extractUrlsFromIndex(sitemapindex: any): SitemapUrl[] {
|
||||
const sitemaps = Array.isArray(sitemapindex.sitemap)
|
||||
? sitemapindex.sitemap
|
||||
: [sitemapindex.sitemap];
|
||||
|
||||
return sitemaps
|
||||
.filter((sitemap: any) => sitemap && sitemap.loc)
|
||||
.map((sitemap: any) => ({
|
||||
loc: sitemap.loc,
|
||||
lastmod: sitemap.lastmod,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract URLs from a regular sitemap urlset
|
||||
*/
|
||||
private extractUrlsFromUrlset(urlset: any): SitemapUrl[] {
|
||||
if (!urlset.url) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const urls = Array.isArray(urlset.url) ? urlset.url : [urlset.url];
|
||||
|
||||
return urls
|
||||
.filter((url: any) => url && url.loc)
|
||||
.map((url: any) => ({
|
||||
loc: url.loc,
|
||||
lastmod: url.lastmod,
|
||||
changefreq: url.changefreq,
|
||||
priority: url.priority,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate if a URL looks like a valid sitemap URL
|
||||
*
|
||||
* @param url - The URL to validate
|
||||
* @returns True if the URL is valid
|
||||
*/
|
||||
static isValidSitemapUrl(url: string): boolean {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
return urlObj.protocol === 'http:' || urlObj.protocol === 'https:';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter URLs to only include likely educational resource URLs
|
||||
* This is a basic heuristic - can be enhanced based on specific requirements
|
||||
*
|
||||
* @param urls - Array of sitemap URLs to filter
|
||||
* @returns Filtered array of URLs
|
||||
*/
|
||||
static filterEducationalUrls(urls: SitemapUrl[]): SitemapUrl[] {
|
||||
return urls.filter(url => {
|
||||
const urlPath = url.loc.toLowerCase();
|
||||
// Basic filtering for common educational content patterns
|
||||
return (
|
||||
urlPath.includes('/course') ||
|
||||
urlPath.includes('/lesson') ||
|
||||
urlPath.includes('/module') ||
|
||||
urlPath.includes('/resource') ||
|
||||
urlPath.includes('/material') ||
|
||||
urlPath.includes('/content') ||
|
||||
urlPath.includes('/oer') ||
|
||||
urlPath.includes('/learning')
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
389
src/cli.ts
Normal file
389
src/cli.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { Command } from 'commander';
|
||||
import { SitemapParser } from './SitemapParser.js';
|
||||
import { PageFetcher } from './PageFetcher.js';
|
||||
import { MetadataExtractor } from './MetadataExtractor.js';
|
||||
import { writeJsonLines, validateOutputPath } from './outputWriter.js';
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('amb-sitemap-parser')
|
||||
.description('Parse sitemaps and extract educational metadata from web pages')
|
||||
.version('0.1.0');
|
||||
|
||||
// Parse command
|
||||
program
|
||||
.command('parse')
|
||||
.description('Parse a sitemap and list all URLs')
|
||||
.argument('<sitemap-url>', 'URL of the sitemap to parse')
|
||||
.option('-l, --limit <number>', 'Limit to first N URLs', parseInt)
|
||||
.option('-p, --pretty', 'Pretty-print JSON output', false)
|
||||
.action(async (sitemapUrl: string, options: { limit?: number; pretty: boolean }) => {
|
||||
try {
|
||||
const parser = new SitemapParser();
|
||||
const sitemap = await parser.parseFromUrl(sitemapUrl);
|
||||
|
||||
let urls = sitemap.urls.map(u => u.loc);
|
||||
|
||||
if (options.limit) {
|
||||
urls = urls.slice(0, options.limit);
|
||||
}
|
||||
|
||||
const output = {
|
||||
urls,
|
||||
isIndex: sitemap.isIndex,
|
||||
count: urls.length,
|
||||
};
|
||||
|
||||
const json = options.pretty
|
||||
? JSON.stringify(output, null, 2)
|
||||
: JSON.stringify(output);
|
||||
|
||||
console.log(json);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : 'Unknown error');
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
// Extract command
|
||||
program
|
||||
.command('extract')
|
||||
.description('Full pipeline: parse sitemap, fetch pages, extract educational metadata')
|
||||
.argument('<sitemap-url>', 'URL of the sitemap to parse')
|
||||
.option('-l, --limit <number>', 'Limit URLs to process', parseInt)
|
||||
.option('-c, --max-concurrency <number>', 'Maximum concurrent requests', parseInt, 5)
|
||||
.option('-t, --timeout <number>', 'Request timeout in milliseconds', parseInt, 30000)
|
||||
.option('-o, --output <filepath>', 'Save metadata to JSONL file (one JSON-LD object per line)')
|
||||
.option('-p, --pretty', 'Pretty-print JSON output', false)
|
||||
.option('--jsonl', 'Output resources as newline-delimited JSON to stdout (one resource per line)', false)
|
||||
.option('-v, --verbose', 'Show progress logs', false)
|
||||
.action(async (sitemapUrl: string, options: {
|
||||
limit?: number;
|
||||
maxConcurrency: number;
|
||||
timeout: number;
|
||||
output?: string;
|
||||
pretty: boolean;
|
||||
jsonl: boolean;
|
||||
verbose: boolean;
|
||||
}) => {
|
||||
try {
|
||||
// Validate mutually exclusive options
|
||||
if (options.output && options.jsonl) {
|
||||
console.error('Error: Cannot use --output and --jsonl together.');
|
||||
console.error('Use --output file.jsonl for file output, or --jsonl for stdout streaming.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Logger function that writes to stderr if verbose is enabled
|
||||
const logger = options.verbose
|
||||
? (message: string, level: 'info' | 'warn' | 'error') => {
|
||||
const prefix = level === 'error' ? '[ERROR]' : level === 'warn' ? '[WARN]' : '[INFO]';
|
||||
console.error(`${prefix} ${message}`);
|
||||
}
|
||||
: undefined;
|
||||
|
||||
// Step 1: Parse sitemap
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Parsing sitemap: ${sitemapUrl}`);
|
||||
}
|
||||
const parser = new SitemapParser({ logger });
|
||||
const sitemap = await parser.parseFromUrl(sitemapUrl);
|
||||
|
||||
let urls = sitemap.urls;
|
||||
if (options.limit) {
|
||||
urls = urls.slice(0, options.limit);
|
||||
}
|
||||
|
||||
const totalUrls = urls.length;
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Found ${totalUrls} URLs to process`);
|
||||
}
|
||||
|
||||
// Step 2: Fetch pages
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Fetching pages with concurrency: ${options.maxConcurrency}`);
|
||||
}
|
||||
const fetcher = new PageFetcher({
|
||||
logger,
|
||||
maxConcurrency: options.maxConcurrency,
|
||||
timeout: options.timeout,
|
||||
});
|
||||
const fetchResults = await fetcher.fetchPages(urls);
|
||||
|
||||
const fetchedCount = fetchResults.filter(r => r.success).length;
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Successfully fetched ${fetchedCount}/${totalUrls} pages`);
|
||||
}
|
||||
|
||||
// Step 3: Extract metadata
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Extracting metadata from fetched pages`);
|
||||
}
|
||||
const extractor = new MetadataExtractor({ logger });
|
||||
const metadata = await extractor.extractFromPages(fetchResults);
|
||||
|
||||
const withMetadataCount = metadata.filter(m => !m.error && (m.jsonLd || m.name || m.description)).length;
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Extracted metadata from ${withMetadataCount}/${fetchedCount} pages`);
|
||||
}
|
||||
|
||||
// Step 4: Filter educational content
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Filtering for educational content`);
|
||||
}
|
||||
const educationalMetadata = MetadataExtractor.filterEducationalMetadata(metadata);
|
||||
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Found ${educationalMetadata.length} pages with educational metadata`);
|
||||
}
|
||||
|
||||
// Step 5: Handle output
|
||||
if (options.jsonl) {
|
||||
// Output JSONL to stdout - one AMB resource per line
|
||||
for (const item of educationalMetadata) {
|
||||
if (item.jsonLd && Array.isArray(item.jsonLd)) {
|
||||
for (const resource of item.jsonLd) {
|
||||
console.log(JSON.stringify(resource));
|
||||
}
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
} else if (options.output) {
|
||||
// Validate output path early
|
||||
try {
|
||||
await validateOutputPath(options.output);
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : 'Invalid output path');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Write JSONL file
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Writing output to ${options.output}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const writeResult = await writeJsonLines(educationalMetadata, options.output);
|
||||
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Wrote ${writeResult.recordsWritten} records to ${options.output}`);
|
||||
}
|
||||
|
||||
// Output summary only (not full metadata)
|
||||
const summary = {
|
||||
summary: {
|
||||
totalUrls,
|
||||
fetched: fetchedCount,
|
||||
withMetadata: withMetadataCount,
|
||||
educational: educationalMetadata.length,
|
||||
recordsWritten: writeResult.recordsWritten,
|
||||
},
|
||||
};
|
||||
|
||||
const json = options.pretty
|
||||
? JSON.stringify(summary, null, 2)
|
||||
: JSON.stringify(summary);
|
||||
|
||||
console.log(json);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Error writing output file:', error instanceof Error ? error.message : 'Unknown error');
|
||||
process.exit(1);
|
||||
}
|
||||
} else {
|
||||
// No output file: existing behavior (full metadata + summary to stdout)
|
||||
const output = {
|
||||
metadata: educationalMetadata,
|
||||
summary: {
|
||||
totalUrls,
|
||||
fetched: fetchedCount,
|
||||
withMetadata: withMetadataCount,
|
||||
educational: educationalMetadata.length,
|
||||
},
|
||||
};
|
||||
|
||||
const json = options.pretty
|
||||
? JSON.stringify(output, null, 2)
|
||||
: JSON.stringify(output);
|
||||
|
||||
console.log(json);
|
||||
process.exit(0);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : 'Unknown error');
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
// Fetch command
|
||||
program
|
||||
.command('fetch')
|
||||
.description('Fetch URL(s) directly and extract educational metadata (no sitemap needed)')
|
||||
.argument('<urls...>', 'One or more URLs to fetch and extract metadata from')
|
||||
.option('-c, --max-concurrency <number>', 'Maximum concurrent requests', parseInt, 5)
|
||||
.option('-t, --timeout <number>', 'Request timeout in milliseconds', parseInt, 30000)
|
||||
.option('-o, --output <filepath>', 'Save metadata to JSONL file (one JSON-LD object per line)')
|
||||
.option('-p, --pretty', 'Pretty-print JSON output', false)
|
||||
.option('--jsonl', 'Output resources as newline-delimited JSON to stdout (one resource per line)', false)
|
||||
.option('-v, --verbose', 'Show progress logs', false)
|
||||
.action(async (urls: string[], options: {
|
||||
maxConcurrency: number;
|
||||
timeout: number;
|
||||
output?: string;
|
||||
pretty: boolean;
|
||||
jsonl: boolean;
|
||||
verbose: boolean;
|
||||
}) => {
|
||||
try {
|
||||
// Validate mutually exclusive options
|
||||
if (options.output && options.jsonl) {
|
||||
console.error('Error: Cannot use --output and --jsonl together.');
|
||||
console.error('Use --output file.jsonl for file output, or --jsonl for stdout streaming.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Logger function that writes to stderr if verbose is enabled
|
||||
const logger = options.verbose
|
||||
? (message: string, level: 'info' | 'warn' | 'error') => {
|
||||
const prefix = level === 'error' ? '[ERROR]' : level === 'warn' ? '[WARN]' : '[INFO]';
|
||||
console.error(`${prefix} ${message}`);
|
||||
}
|
||||
: undefined;
|
||||
|
||||
// Validate URLs
|
||||
const invalidUrls = urls.filter(url => !PageFetcher.isValidUrl(url));
|
||||
if (invalidUrls.length > 0) {
|
||||
console.error('Error: Invalid URLs provided:');
|
||||
invalidUrls.forEach(url => console.error(` - ${url}`));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Convert URLs to SitemapUrl format
|
||||
const sitemapUrls = urls.map(url => ({ loc: url }));
|
||||
const totalUrls = sitemapUrls.length;
|
||||
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Processing ${totalUrls} URL(s)`);
|
||||
}
|
||||
|
||||
// Step 1: Fetch pages
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Fetching pages with concurrency: ${options.maxConcurrency}`);
|
||||
}
|
||||
const fetcher = new PageFetcher({
|
||||
logger,
|
||||
maxConcurrency: options.maxConcurrency,
|
||||
timeout: options.timeout,
|
||||
});
|
||||
const fetchResults = await fetcher.fetchPages(sitemapUrls);
|
||||
|
||||
const fetchedCount = fetchResults.filter(r => r.success).length;
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Successfully fetched ${fetchedCount}/${totalUrls} pages`);
|
||||
}
|
||||
|
||||
// Step 2: Extract metadata
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Extracting metadata from fetched pages`);
|
||||
}
|
||||
const extractor = new MetadataExtractor({ logger });
|
||||
const metadata = await extractor.extractFromPages(fetchResults);
|
||||
|
||||
const withMetadataCount = metadata.filter(m => !m.error && (m.jsonLd || m.name || m.description)).length;
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Extracted metadata from ${withMetadataCount}/${fetchedCount} pages`);
|
||||
}
|
||||
|
||||
// Step 3: Filter educational content
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Filtering for educational content`);
|
||||
}
|
||||
const educationalMetadata = MetadataExtractor.filterEducationalMetadata(metadata);
|
||||
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Found ${educationalMetadata.length} pages with educational metadata`);
|
||||
}
|
||||
|
||||
// Step 4: Handle output
|
||||
if (options.jsonl) {
|
||||
// Output JSONL to stdout - one AMB resource per line
|
||||
for (const item of educationalMetadata) {
|
||||
if (item.jsonLd && Array.isArray(item.jsonLd)) {
|
||||
for (const resource of item.jsonLd) {
|
||||
console.log(JSON.stringify(resource));
|
||||
}
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
} else if (options.output) {
|
||||
// Validate output path early
|
||||
try {
|
||||
await validateOutputPath(options.output);
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : 'Invalid output path');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Write JSONL file
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Writing output to ${options.output}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const writeResult = await writeJsonLines(educationalMetadata, options.output);
|
||||
|
||||
if (options.verbose) {
|
||||
console.error(`[INFO] Wrote ${writeResult.recordsWritten} records to ${options.output}`);
|
||||
}
|
||||
|
||||
// Output summary only (not full metadata)
|
||||
const summary = {
|
||||
summary: {
|
||||
totalUrls,
|
||||
fetched: fetchedCount,
|
||||
withMetadata: withMetadataCount,
|
||||
educational: educationalMetadata.length,
|
||||
recordsWritten: writeResult.recordsWritten,
|
||||
},
|
||||
};
|
||||
|
||||
const json = options.pretty
|
||||
? JSON.stringify(summary, null, 2)
|
||||
: JSON.stringify(summary);
|
||||
|
||||
console.log(json);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Error writing output file:', error instanceof Error ? error.message : 'Unknown error');
|
||||
process.exit(1);
|
||||
}
|
||||
} else {
|
||||
// No output file: existing behavior (full metadata + summary to stdout)
|
||||
const output = {
|
||||
metadata: educationalMetadata,
|
||||
summary: {
|
||||
totalUrls,
|
||||
fetched: fetchedCount,
|
||||
withMetadata: withMetadataCount,
|
||||
educational: educationalMetadata.length,
|
||||
},
|
||||
};
|
||||
|
||||
const json = options.pretty
|
||||
? JSON.stringify(output, null, 2)
|
||||
: JSON.stringify(output);
|
||||
|
||||
console.log(json);
|
||||
process.exit(0);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error:', error instanceof Error ? error.message : 'Unknown error');
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
program.parse();
|
||||
27
src/index.ts
Normal file
27
src/index.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* amb-sitemap-parser
|
||||
*
|
||||
* A library for parsing sitemaps and extracting educational metadata from web pages
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
// Export all classes
|
||||
export { SitemapParser } from './SitemapParser.js';
|
||||
export type { SitemapParserOptions } from './SitemapParser.js';
|
||||
|
||||
export { PageFetcher } from './PageFetcher.js';
|
||||
export type { PageFetcherOptions } from './PageFetcher.js';
|
||||
|
||||
export { MetadataExtractor } from './MetadataExtractor.js';
|
||||
export type { MetadataExtractorOptions } from './MetadataExtractor.js';
|
||||
|
||||
// Export all types
|
||||
export type {
|
||||
SitemapUrl,
|
||||
ParsedSitemap,
|
||||
FetchResult,
|
||||
FetchOptions,
|
||||
ExtractedMetadata,
|
||||
LoggerFunction,
|
||||
} from './types.js';
|
||||
85
src/outputWriter.ts
Normal file
85
src/outputWriter.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { promises as fs } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import type { ExtractedMetadata } from './types.js';
|
||||
|
||||
/**
|
||||
* Result of writing JSONL output
|
||||
*/
|
||||
export interface WriteResult {
|
||||
recordsWritten: number;
|
||||
filepath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that the output path is valid and writable
|
||||
*
|
||||
* @param filepath - Path to validate
|
||||
* @throws Error if path is invalid or directory doesn't exist
|
||||
*/
|
||||
export async function validateOutputPath(filepath: string): Promise<void> {
|
||||
if (!filepath || filepath.trim() === '') {
|
||||
throw new Error('Output filepath cannot be empty');
|
||||
}
|
||||
|
||||
// Check if parent directory exists
|
||||
const dir = dirname(filepath);
|
||||
try {
|
||||
await fs.access(dir);
|
||||
} catch (error) {
|
||||
throw new Error(`Output directory does not exist: ${dir}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes extracted metadata to a JSON Lines (JSONL) file
|
||||
*
|
||||
* Flattens the metadata array so that each JSON-LD object becomes a separate line.
|
||||
* Each line contains:
|
||||
* - url: The source URL of the page
|
||||
* - jsonLd: A single JSON-LD object from that page
|
||||
*
|
||||
* This format is ideal for streaming and piping to other tools.
|
||||
*
|
||||
* @param metadata - Array of extracted metadata
|
||||
* @param filepath - Path to write the JSONL file
|
||||
* @returns Result containing count of records written
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const result = await writeJsonLines(metadata, 'output.jsonl');
|
||||
* console.log(`Wrote ${result.recordsWritten} records`);
|
||||
* ```
|
||||
*/
|
||||
export async function writeJsonLines(
|
||||
metadata: ExtractedMetadata[],
|
||||
filepath: string
|
||||
): Promise<WriteResult> {
|
||||
// Validate path before processing
|
||||
await validateOutputPath(filepath);
|
||||
|
||||
const lines: string[] = [];
|
||||
|
||||
// Flatten: one line per JSON-LD object
|
||||
for (const item of metadata) {
|
||||
if (!item.jsonLd || item.jsonLd.length === 0) {
|
||||
continue; // Skip items without JSON-LD data
|
||||
}
|
||||
|
||||
for (const jsonLdObject of item.jsonLd) {
|
||||
const record = {
|
||||
url: item.url,
|
||||
jsonLd: jsonLdObject,
|
||||
};
|
||||
lines.push(JSON.stringify(record));
|
||||
}
|
||||
}
|
||||
|
||||
// Write all lines with newline separator
|
||||
const content = lines.length > 0 ? lines.join('\n') + '\n' : '';
|
||||
await fs.writeFile(filepath, content, 'utf-8');
|
||||
|
||||
return {
|
||||
recordsWritten: lines.length,
|
||||
filepath,
|
||||
};
|
||||
}
|
||||
56
src/types.ts
Normal file
56
src/types.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Represents a URL entry in a sitemap
|
||||
*/
|
||||
export interface SitemapUrl {
|
||||
loc: string;
|
||||
lastmod?: string;
|
||||
changefreq?: string;
|
||||
priority?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of parsing a sitemap
|
||||
*/
|
||||
export interface ParsedSitemap {
|
||||
urls: SitemapUrl[];
|
||||
isIndex: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of fetching a single page
|
||||
*/
|
||||
export interface FetchResult {
|
||||
url: string;
|
||||
success: boolean;
|
||||
html?: string;
|
||||
error?: string;
|
||||
statusCode?: number;
|
||||
responseTime?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for configuring the page fetcher
|
||||
*/
|
||||
export interface FetchOptions {
|
||||
timeout?: number;
|
||||
maxConcurrency?: number;
|
||||
userAgent?: string;
|
||||
delayBetweenRequests?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata extracted from a web page
|
||||
*/
|
||||
export interface ExtractedMetadata {
|
||||
url: string;
|
||||
jsonLd?: any[];
|
||||
name?: string;
|
||||
description?: string;
|
||||
extractionTime: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Logger function type for optional logging
|
||||
*/
|
||||
export type LoggerFunction = (message: string, level: 'info' | 'warn' | 'error') => void;
|
||||
22
tsconfig.json
Normal file
22
tsconfig.json
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"lib": ["ES2022"],
|
||||
"moduleResolution": "bundler",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"noEmit": false
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/*.test.ts"]
|
||||
}
|
||||
25
vitest.config.ts
Normal file
25
vitest.config.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
reporter: ['text', 'json', 'html'],
|
||||
exclude: [
|
||||
'node_modules/',
|
||||
'dist/',
|
||||
'**/*.d.ts',
|
||||
'**/*.config.*',
|
||||
'**/index.ts'
|
||||
],
|
||||
thresholds: {
|
||||
lines: 80,
|
||||
functions: 80,
|
||||
branches: 80,
|
||||
statements: 80
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue