summaryrefslogtreecommitdiff
path: root/app/lib/scraper-runner.server.ts
diff options
context:
space:
mode:
authoryyamashita <yyamashita@mosquit.one>2026-05-06 22:07:53 +0900
committeryyamashita <yyamashita@mosquit.one>2026-05-06 22:07:53 +0900
commitbe55729482296663da8c96723bfd22080e6762c1 (patch)
treefcd94b1dc5c55f3a80796c90a555863d13fc9a95 /app/lib/scraper-runner.server.ts
parent014b29bc22b1c207a03dd560051ecdd5df63f0b1 (diff)
Add Tokyo livehouse event aggregator service
Full-stack React Router v7 app that scrapes event listings from major Tokyo live venues (Liquid Room, WWW/WWW X, Shibuya O-EAST, Shinjuku LOFT, Club Quattro) and stores them in SQLite for browsing and search. - Modular scraper architecture: add a new venue by dropping a file in app/scrapers/ and registering it in index.ts - Routes: /events (filter by keyword/venue/date), /events/:id, /venues, GET /api/scrape - EventCard shows artist, date/time, venue, ticket URL, and fee - Post-scrape per-venue Markdown files generated to events/ (dev reference) - /add-livehouse Claude Code skill defined in .claude/commands/ Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'app/lib/scraper-runner.server.ts')
-rw-r--r--app/lib/scraper-runner.server.ts77
1 files changed, 77 insertions, 0 deletions
diff --git a/app/lib/scraper-runner.server.ts b/app/lib/scraper-runner.server.ts
new file mode 100644
index 0000000..070a568
--- /dev/null
+++ b/app/lib/scraper-runner.server.ts
@@ -0,0 +1,77 @@
+import { upsertVenue, upsertEvent } from "./db.server";
+import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server";
+import { ALL_SCRAPERS } from "~/scrapers/index";
+
+export interface ScrapeResult {
+ venue_id: string;
+ venue_name: string;
+ events_saved: number;
+ markdown_path?: string;
+ error?: string;
+}
+
+export async function runAllScrapers(): Promise<ScrapeResult[]> {
+ const results: ScrapeResult[] = [];
+ const successIds: string[] = [];
+
+ for (const scraper of ALL_SCRAPERS) {
+ const { venue } = scraper;
+ upsertVenue(venue.id, venue.name, venue.url, venue.area);
+
+ try {
+ const events = await scraper.scrape();
+ for (const event of events) {
+ upsertEvent(event);
+ }
+ successIds.push(venue.id);
+ results.push({
+ venue_id: venue.id,
+ venue_name: venue.name,
+ events_saved: events.length,
+ });
+ } catch (err) {
+ results.push({
+ venue_id: venue.id,
+ venue_name: venue.name,
+ events_saved: 0,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ }
+ }
+
+ // Generate Markdown files for all venues that scraped successfully
+ generateAllVenueMarkdown(successIds);
+
+ return results;
+}
+
+export async function runScraper(venueId: string): Promise<ScrapeResult> {
+ const scraper = ALL_SCRAPERS.find((s) => s.venue.id === venueId);
+ if (!scraper) {
+ return { venue_id: venueId, venue_name: venueId, events_saved: 0, error: "Scraper not found" };
+ }
+
+ const { venue } = scraper;
+ upsertVenue(venue.id, venue.name, venue.url, venue.area);
+
+ try {
+ const events = await scraper.scrape();
+ for (const event of events) {
+ upsertEvent(event);
+ }
+ generateVenueMarkdown(venue.id);
+ return {
+ venue_id: venue.id,
+ venue_name: venue.name,
+ events_saved: events.length,
+ markdown_path: `events/${venue.id}.md`,
+ };
+ } catch (err) {
+ return {
+ venue_id: venue.id,
+ venue_name: venue.name,
+ events_saved: 0,
+ error: err instanceof Error ? err.message : String(err),
+ };
+ }
+}