diff options
| author | yyamashita <yyamashita@mosquit.one> | 2026-05-07 10:16:43 +0900 |
|---|---|---|
| committer | yyamashita <yyamashita@mosquit.one> | 2026-05-07 10:16:43 +0900 |
| commit | 0cd5fb770ca9bd3f304d9556a4b33a4ad4f45e7e (patch) | |
| tree | aa316c21d7195689d87669338373d83b0b6ac3fb /app/lib | |
| parent | 538fd636e25595d88a958344d285c0e7cf44e530 (diff) | |
Playwright scraping for FLAT/Pitbar; web UI display-only
- Install Playwright + Chromium; add shared browser singleton (playwright.server.ts)
- Rewrite flat-nishiogikubo scraper: Wix calendar via headless browser,
month navigation via date picker, extracts .WPczEB/.ExCBIq selectors
- Rewrite pitbar-nishiogikubo scraper: freecalend.com via headless browser,
parses cal-{member}-{year}-{month}-{day} cell IDs
- scraper-runner: close shared browser after each run with closeBrowser()
- Remove all scrape trigger buttons from web UI (events index, venues page);
remove /api/scrape and /api/scrape-status from routes.ts
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'app/lib')
| -rw-r--r-- | app/lib/playwright.server.ts | 17 | ||||
| -rw-r--r-- | app/lib/scraper-runner.server.ts | 5 |
2 files changed, 22 insertions, 0 deletions
diff --git a/app/lib/playwright.server.ts b/app/lib/playwright.server.ts new file mode 100644 index 0000000..aadb5d3 --- /dev/null +++ b/app/lib/playwright.server.ts @@ -0,0 +1,17 @@ +import { chromium, type Browser } from "playwright"; + +let _browser: Browser | null = null; + +export async function getBrowser(): Promise<Browser> { + if (!_browser || !_browser.isConnected()) { + _browser = await chromium.launch({ headless: true }); + } + return _browser; +} + +export async function closeBrowser(): Promise<void> { + if (_browser) { + await _browser.close(); + _browser = null; + } +} diff --git a/app/lib/scraper-runner.server.ts b/app/lib/scraper-runner.server.ts index 87dd16c..012ff95 100644 --- a/app/lib/scraper-runner.server.ts +++ b/app/lib/scraper-runner.server.ts @@ -7,6 +7,7 @@ import { type ScrapeLog, } from "./db.server"; import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server"; +import { closeBrowser } from "./playwright.server"; import { ALL_SCRAPERS } from "~/scrapers/index"; import type { EventInput } from "./db.server"; @@ -77,6 +78,8 @@ export async function runAllScrapers(run_id = randomUUID()): Promise<ScrapeResul } generateAllVenueMarkdown(successIds); + // Close shared Playwright browser if it was opened by any scraper + await closeBrowser(); return results; } @@ -104,5 +107,7 @@ export async function runScraper(venueId: string, run_id = randomUUID()): Promis const error = err instanceof Error ? err.message : String(err); updateScrapeLog(logId, "error", 0, error); return { run_id, venue_id: venue.id, venue_name: venue.name, status: "error", events_saved: 0, error }; + } finally { + await closeBrowser(); } } |
