Crawl a website

const baseUrl = process.env.HORIZON_BASE_URL ?? 'https://api.horizon.new/v1';

const crawlResponse = await fetchWithPayment(`${baseUrl}/crawl`, {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({
    url: 'https://docs.horizon.new/api-reference/endpoint',
    depth: 1,
    priority: 'high',
    notify: {
      url: 'https://example.com/webhooks/crawl',
      events: ['completed'],
    },
  }),
});

const crawlJob = await crawlResponse.json();
console.log('crawl', crawlJob.crawlId, crawlJob.jobId, crawlJob.statusUrl);

if (crawlJob.status === 'completed' && crawlJob.result) {
  console.log('Captured pages immediately', crawlJob.result.pages.length);
  // You can skip polling if the crawl finished inline.
}

let status;

do {
  status = await fetchWithPayment(crawlJob.statusUrl).then((res) => res.json());

  if (status.state === 'processing') {
    console.log('Still walking links...', status.progress?.percentComplete ?? 0);
    await new Promise((resolve) => setTimeout(resolve, 5000));
  }
} while (status.state === 'processing');

if (status.state !== 'succeeded') {
  throw new Error(`Crawl failed: ${status.error?.code ?? 'unknown'}`);
}

console.log('Captured pages', status.result.pages.length);

import crypto from 'node:crypto';

const verifyWebhookSignature = (signature: string, payload: unknown) => {
  const secret = process.env.HORIZON_WEBHOOK_SECRET!;
  const computed = crypto
    .createHmac('sha256', secret)
    .update(JSON.stringify(payload))
    .digest('hex');

  return crypto.timingSafeEqual(Buffer.from(signature, 'hex'), Buffer.from(computed, 'hex'));
};

app.post('/webhooks/crawl', async (req, res) => {
  const signature = req.header('X-Horizon-Signature') ?? '';

  if (!verifyWebhookSignature(signature, req.body)) {
    return res.status(401).end();
  }

  const { jobId, state, result } = req.body;
  if (state === 'succeeded') {
    console.log('Crawl completed', jobId, result.pages.length);
  }

  res.status(204).end();
});