import { upsertVenue, upsertEvent } from "./db.server"; import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server"; import { ALL_SCRAPERS } from "~/scrapers/index"; import type { EventInput } from "./db.server"; const SCRAPE_WINDOW_DAYS = 35; // ~1 month function scrapeWindow(): { from: string; to: string } { const from = new Date(); from.setHours(0, 0, 0, 0); const to = new Date(from); to.setDate(to.getDate() + SCRAPE_WINDOW_DAYS); return { from: from.toISOString().slice(0, 10), to: to.toISOString().slice(0, 10), }; } function withinWindow(event: EventInput, from: string, to: string): boolean { return event.date >= from && event.date <= to; } export interface ScrapeResult { venue_id: string; venue_name: string; events_saved: number; markdown_path?: string; error?: string; } export async function runAllScrapers(): Promise { const results: ScrapeResult[] = []; const successIds: string[] = []; for (const scraper of ALL_SCRAPERS) { const { venue } = scraper; upsertVenue(venue.id, venue.name, venue.url, venue.area); try { const { from, to } = scrapeWindow(); const events = (await scraper.scrape()).filter((e) => withinWindow(e, from, to) ); for (const event of events) { upsertEvent(event); } successIds.push(venue.id); results.push({ venue_id: venue.id, venue_name: venue.name, events_saved: events.length, }); } catch (err) { results.push({ venue_id: venue.id, venue_name: venue.name, events_saved: 0, error: err instanceof Error ? err.message : String(err), }); } } // Generate Markdown files for all venues that scraped successfully generateAllVenueMarkdown(successIds); return results; } export async function runScraper(venueId: string): Promise { const scraper = ALL_SCRAPERS.find((s) => s.venue.id === venueId); if (!scraper) { return { venue_id: venueId, venue_name: venueId, events_saved: 0, error: "Scraper not found" }; } const { venue } = scraper; upsertVenue(venue.id, venue.name, venue.url, venue.area); try { const { from, to } = scrapeWindow(); const events = (await scraper.scrape()).filter((e) => withinWindow(e, from, to) ); for (const event of events) { upsertEvent(event); } generateVenueMarkdown(venue.id); return { venue_id: venue.id, venue_name: venue.name, events_saved: events.length, markdown_path: `events/${venue.id}.md`, }; } catch (err) { return { venue_id: venue.id, venue_name: venue.name, events_saved: 0, error: err instanceof Error ? err.message : String(err), }; } }