llm-scraper/examples/codegen.ts

import { chromium } from 'playwright'
import { z } from 'zod'
import { openai } from '@ai-sdk/openai'
import LLMScraper from './../src'

// Launch a browser instance
const browser = await chromium.launch()

// Initialize LLM provider
const llm = openai('gpt-4o-mini')

// Create a new LLMScraper
const scraper = new LLMScraper(llm)

// Open new page
const page = await browser.newPage()
await page.goto('https://news.ycombinator.com')

// Define schema to extract contents into
const schema = z.object({
  top: z
    .array(
      z.object({
        title: z.string(),
        points: z.number(),
        by: z.string(),
        commentsURL: z.string(),
      })
    )
    .length(5)
    .describe('Top 5 stories on Hacker News'),
})

// Generate code and run it on the page
const { code } = await scraper.generate(page, schema, {
  format: 'raw_html',
})
console.log('code', code)

const result = await page.evaluate(code)
const data = schema.parse(result)

// Show the parsed result
console.log('result', data)

await page.close()
await browser.close()