summaryrefslogtreecommitdiff
path: root/app/lib/scraper-runner.server.ts
diff options
context:
space:
mode:
authoryyamashita <yyamashita@mosquit.one>2026-05-06 22:09:41 +0900
committeryyamashita <yyamashita@mosquit.one>2026-05-06 22:09:41 +0900
commitc18885f2b022f4a8116809aa893929f2c278b319 (patch)
treec2818ef46ff58c7e3bc4cb2bd9b8a48e5821c3f8 /app/lib/scraper-runner.server.ts
parentbe55729482296663da8c96723bfd22080e6762c1 (diff)
Limit scrape window and default event list to ~1 month
Scraper runner now filters events to today + 35 days before upserting, so the DB only holds the upcoming month of events. Event list loader defaults date_from/date_to to the same window when no query params are set, and passes the values down to FilterBar so the date inputs are pre-filled on first load. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'app/lib/scraper-runner.server.ts')
-rw-r--r--app/lib/scraper-runner.server.ts28
1 files changed, 26 insertions, 2 deletions
diff --git a/app/lib/scraper-runner.server.ts b/app/lib/scraper-runner.server.ts
index 070a568..191dd00 100644
--- a/app/lib/scraper-runner.server.ts
+++ b/app/lib/scraper-runner.server.ts
@@ -1,6 +1,24 @@
import { upsertVenue, upsertEvent } from "./db.server";
import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server";
import { ALL_SCRAPERS } from "~/scrapers/index";
+import type { EventInput } from "./db.server";
+
+const SCRAPE_WINDOW_DAYS = 35; // ~1 month
+
+function scrapeWindow(): { from: string; to: string } {
+ const from = new Date();
+ from.setHours(0, 0, 0, 0);
+ const to = new Date(from);
+ to.setDate(to.getDate() + SCRAPE_WINDOW_DAYS);
+ return {
+ from: from.toISOString().slice(0, 10),
+ to: to.toISOString().slice(0, 10),
+ };
+}
+
+function withinWindow(event: EventInput, from: string, to: string): boolean {
+ return event.date >= from && event.date <= to;
+}
export interface ScrapeResult {
venue_id: string;
@@ -19,7 +37,10 @@ export async function runAllScrapers(): Promise<ScrapeResult[]> {
upsertVenue(venue.id, venue.name, venue.url, venue.area);
try {
- const events = await scraper.scrape();
+ const { from, to } = scrapeWindow();
+ const events = (await scraper.scrape()).filter((e) =>
+ withinWindow(e, from, to)
+ );
for (const event of events) {
upsertEvent(event);
}
@@ -55,7 +76,10 @@ export async function runScraper(venueId: string): Promise<ScrapeResult> {
upsertVenue(venue.id, venue.name, venue.url, venue.area);
try {
- const events = await scraper.scrape();
+ const { from, to } = scrapeWindow();
+ const events = (await scraper.scrape()).filter((e) =>
+ withinWindow(e, from, to)
+ );
for (const event of events) {
upsertEvent(event);
}