diff options
| author | yyamashita <yyamashita@mosquit.one> | 2026-05-06 22:24:38 +0900 |
|---|---|---|
| committer | yyamashita <yyamashita@mosquit.one> | 2026-05-06 22:24:38 +0900 |
| commit | 538fd636e25595d88a958344d285c0e7cf44e530 (patch) | |
| tree | eb2999f355570224fa96877d5043af2ef3ec76ef /scripts | |
| parent | f817604858891edb79e26459dae884b158774db1 (diff) | |
Async scraping, scrape_logs, and CLI
Background scraping:
- POST /api/scrape returns 202 immediately with run_id; scraping runs async
- GET /api/scrape-status?run_id=xxx polls for results per venue
- scrape_logs table: per-venue status (running/ok/error), events_saved, error, timestamps
CLI (npm run scrape):
- npm run scrape — 全会場をスクレイプ、結果を色付きで出力
- npm run scrape liquid-room — 特定会場のみ
- npm run scrape -- --list — 登録済み会場一覧を表示
- エラー時は exit code 1 + エラーメッセージを dim 表示
Venues page:
- 最終スクレイプ日時・成否をインラインで表示
- 会場ごとの「更新」ボタンを追加
Bug fix: upsertEvent に description/optional fields のデフォルト値を設定し
better-sqlite3 の "Missing named parameter" エラーを解消
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/scrape.ts | 67 |
1 files changed, 67 insertions, 0 deletions
diff --git a/scripts/scrape.ts b/scripts/scrape.ts new file mode 100644 index 0000000..62e9172 --- /dev/null +++ b/scripts/scrape.ts @@ -0,0 +1,67 @@ +#!/usr/bin/env node +/** + * CLI スクレイパー + * + * 使い方: + * npm run scrape # 全会場 + * npm run scrape liquid-room # 特定会場 + * npm run scrape -- --list # 登録済み会場を表示 + */ +import { styleText } from "node:util"; +import { runAllScrapers, runScraper } from "../app/lib/scraper-runner.server.js"; +import { ALL_SCRAPERS } from "../app/scrapers/index.js"; + +const args = process.argv.slice(2); +const venueId = args.find((a) => !a.startsWith("--")); +const flagList = args.includes("--list"); + +if (flagList) { + console.log(styleText("bold", "\n登録済みスクレイパー一覧:\n")); + for (const s of ALL_SCRAPERS) { + console.log(` ${styleText("cyan", s.venue.id.padEnd(25))} ${s.venue.name} (${s.venue.area})`); + } + console.log(); + process.exit(0); +} + +const startTime = Date.now(); + +console.log( + styleText("bold", "\n🎸 東京ライブハウス スクレイパー\n") + + (venueId + ? ` 対象: ${styleText("cyan", venueId)}\n` + : ` 対象: ${styleText("cyan", `全 ${ALL_SCRAPERS.length} 会場`)}\n`) +); + +const results = venueId + ? [await runScraper(venueId)] + : await runAllScrapers(); + +const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); +let totalSaved = 0; +let hasError = false; + +for (const r of results) { + const icon = r.status === "ok" ? styleText("green", "✔") : styleText("red", "✖"); + const name = r.venue_name.padEnd(24); + + if (r.status === "ok") { + const count = styleText("green", `${r.events_saved} 件`); + console.log(` ${icon} ${name} ${count}`); + totalSaved += r.events_saved; + } else { + hasError = true; + console.log(` ${icon} ${styleText("red", name)}`); + if (r.error) { + const lines = r.error.split("\n"); + for (const line of lines) { + console.log(` ${styleText("dim", line)}`); + } + } + } +} + +const summary = `\n 合計 ${styleText("bold", `${totalSaved} 件`)} 保存 (${elapsed}s)`; +console.log(summary + "\n"); + +process.exit(hasError ? 1 : 0); |
