POST
/api/site-crawl$0.02🌐 Site Crawler
Crawl a website starting from a given URL, following internal links up to a configurable depth. Returns structured content from each page as Markdown, plus all discovered links. Stays within the same domain. Ideal for building knowledge bases, site audits, and content extraction pipelines.
Request Body
Response Fields
Code Examples
cURL
bash
curl -X POST https://sparkforge.sh/api/site-crawl \
-H "Content-Type: application/json" \
-H "X-PAYMENT: <x402-payment-token>" \
-d '{
"url": "https://sparkforge.sh",
"maxPages": 10,
"depth": 2
}'Python
python
import requests
response = requests.post(
"https://sparkforge.sh/api/site-crawl",
headers={"Content-Type": "application/json", "X-PAYMENT": "<token>"},
json={"url": "https://sparkforge.sh", "maxPages": 5, "depth": 2}
)
data = response.json()
print(f"Crawled {data['totalPages']} pages")
for page in data["pages"]:
print(f" {page['url']} — {page['title']}")
print(f" Links found: {len(page['links'])}")TypeScript / Node.js
typescript
const res = await fetch402("https://sparkforge.sh/api/site-crawl", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url: "https://sparkforge.sh", maxPages: 10, depth: 2 }),
});
const { pages, totalPages } = await res.json();
pages.forEach((p: any) => {
console.log(p.url, p.title);
// p.content contains the page as Markdown
});Example Response
json
{
"pages": [
{
"url": "https://sparkforge.sh",
"title": "SparkForge — AI Agent APIs",
"content": "# SparkForge\n\nPay-per-use APIs for AI agents...",
"links": ["https://sparkforge.sh/docs", "https://sparkforge.sh/api/models"],
"depth": 0,
"statusCode": 200
},
{
"url": "https://sparkforge.sh/docs",
"title": "SparkForge API Docs",
"content": "# Documentation\n\nOne Wallet. 33 Tools...",
"links": [],
"depth": 1,
"statusCode": 200
}
],
"totalPages": 2,
"startUrl": "https://sparkforge.sh"
}💳 Payment via x402
This endpoint costs $0.02 per call, paid in USDC on Base, Polygon, or Solana. Use the @x402/fetch library to handle payments automatically, or implement the x402 handshake manually.
bash
# Free demo mode (no payment)
curl https://sparkforge.sh/api/site-crawl?demo=true
# Or try via MCP (Claude Desktop, Cursor, Windsurf)
# Add to mcp config: { "url": "https://sparkforge.sh/api/mcp" }