diff options
| author | yyamashita <yyamashita@mosquit.one> | 2026-05-06 22:07:53 +0900 |
|---|---|---|
| committer | yyamashita <yyamashita@mosquit.one> | 2026-05-06 22:07:53 +0900 |
| commit | be55729482296663da8c96723bfd22080e6762c1 (patch) | |
| tree | fcd94b1dc5c55f3a80796c90a555863d13fc9a95 /app/lib/scraper-runner.server.ts | |
| parent | 014b29bc22b1c207a03dd560051ecdd5df63f0b1 (diff) | |
Add Tokyo livehouse event aggregator service
Full-stack React Router v7 app that scrapes event listings from major
Tokyo live venues (Liquid Room, WWW/WWW X, Shibuya O-EAST, Shinjuku LOFT,
Club Quattro) and stores them in SQLite for browsing and search.
- Modular scraper architecture: add a new venue by dropping a file in
app/scrapers/ and registering it in index.ts
- Routes: /events (filter by keyword/venue/date), /events/:id, /venues,
GET /api/scrape
- EventCard shows artist, date/time, venue, ticket URL, and fee
- Post-scrape per-venue Markdown files generated to events/ (dev reference)
- /add-livehouse Claude Code skill defined in .claude/commands/
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'app/lib/scraper-runner.server.ts')
| -rw-r--r-- | app/lib/scraper-runner.server.ts | 77 |
1 files changed, 77 insertions, 0 deletions
diff --git a/app/lib/scraper-runner.server.ts b/app/lib/scraper-runner.server.ts new file mode 100644 index 0000000..070a568 --- /dev/null +++ b/app/lib/scraper-runner.server.ts @@ -0,0 +1,77 @@ +import { upsertVenue, upsertEvent } from "./db.server"; +import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server"; +import { ALL_SCRAPERS } from "~/scrapers/index"; + +export interface ScrapeResult { + venue_id: string; + venue_name: string; + events_saved: number; + markdown_path?: string; + error?: string; +} + +export async function runAllScrapers(): Promise<ScrapeResult[]> { + const results: ScrapeResult[] = []; + const successIds: string[] = []; + + for (const scraper of ALL_SCRAPERS) { + const { venue } = scraper; + upsertVenue(venue.id, venue.name, venue.url, venue.area); + + try { + const events = await scraper.scrape(); + for (const event of events) { + upsertEvent(event); + } + successIds.push(venue.id); + results.push({ + venue_id: venue.id, + venue_name: venue.name, + events_saved: events.length, + }); + } catch (err) { + results.push({ + venue_id: venue.id, + venue_name: venue.name, + events_saved: 0, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + // Generate Markdown files for all venues that scraped successfully + generateAllVenueMarkdown(successIds); + + return results; +} + +export async function runScraper(venueId: string): Promise<ScrapeResult> { + const scraper = ALL_SCRAPERS.find((s) => s.venue.id === venueId); + if (!scraper) { + return { venue_id: venueId, venue_name: venueId, events_saved: 0, error: "Scraper not found" }; + } + + const { venue } = scraper; + upsertVenue(venue.id, venue.name, venue.url, venue.area); + + try { + const events = await scraper.scrape(); + for (const event of events) { + upsertEvent(event); + } + generateVenueMarkdown(venue.id); + return { + venue_id: venue.id, + venue_name: venue.name, + events_saved: events.length, + markdown_path: `events/${venue.id}.md`, + }; + } catch (err) { + return { + venue_id: venue.id, + venue_name: venue.name, + events_saved: 0, + error: err instanceof Error ? err.message : String(err), + }; + } +} |
