diff options
| author | yyamashita <yyamashita@mosquit.one> | 2026-05-06 22:09:41 +0900 |
|---|---|---|
| committer | yyamashita <yyamashita@mosquit.one> | 2026-05-06 22:09:41 +0900 |
| commit | c18885f2b022f4a8116809aa893929f2c278b319 (patch) | |
| tree | c2818ef46ff58c7e3bc4cb2bd9b8a48e5821c3f8 /app/lib/scraper-runner.server.ts | |
| parent | be55729482296663da8c96723bfd22080e6762c1 (diff) | |
Limit scrape window and default event list to ~1 month
Scraper runner now filters events to today + 35 days before upserting,
so the DB only holds the upcoming month of events.
Event list loader defaults date_from/date_to to the same window when
no query params are set, and passes the values down to FilterBar so the
date inputs are pre-filled on first load.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'app/lib/scraper-runner.server.ts')
| -rw-r--r-- | app/lib/scraper-runner.server.ts | 28 |
1 files changed, 26 insertions, 2 deletions
diff --git a/app/lib/scraper-runner.server.ts b/app/lib/scraper-runner.server.ts index 070a568..191dd00 100644 --- a/app/lib/scraper-runner.server.ts +++ b/app/lib/scraper-runner.server.ts @@ -1,6 +1,24 @@ import { upsertVenue, upsertEvent } from "./db.server"; import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server"; import { ALL_SCRAPERS } from "~/scrapers/index"; +import type { EventInput } from "./db.server"; + +const SCRAPE_WINDOW_DAYS = 35; // ~1 month + +function scrapeWindow(): { from: string; to: string } { + const from = new Date(); + from.setHours(0, 0, 0, 0); + const to = new Date(from); + to.setDate(to.getDate() + SCRAPE_WINDOW_DAYS); + return { + from: from.toISOString().slice(0, 10), + to: to.toISOString().slice(0, 10), + }; +} + +function withinWindow(event: EventInput, from: string, to: string): boolean { + return event.date >= from && event.date <= to; +} export interface ScrapeResult { venue_id: string; @@ -19,7 +37,10 @@ export async function runAllScrapers(): Promise<ScrapeResult[]> { upsertVenue(venue.id, venue.name, venue.url, venue.area); try { - const events = await scraper.scrape(); + const { from, to } = scrapeWindow(); + const events = (await scraper.scrape()).filter((e) => + withinWindow(e, from, to) + ); for (const event of events) { upsertEvent(event); } @@ -55,7 +76,10 @@ export async function runScraper(venueId: string): Promise<ScrapeResult> { upsertVenue(venue.id, venue.name, venue.url, venue.area); try { - const events = await scraper.scrape(); + const { from, to } = scrapeWindow(); + const events = (await scraper.scrape()).filter((e) => + withinWindow(e, from, to) + ); for (const event of events) { upsertEvent(event); } |
