summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authoryyamashita <yyamashita@mosquit.one>2026-05-06 22:24:38 +0900
committeryyamashita <yyamashita@mosquit.one>2026-05-06 22:24:38 +0900
commit538fd636e25595d88a958344d285c0e7cf44e530 (patch)
treeeb2999f355570224fa96877d5043af2ef3ec76ef /scripts
parentf817604858891edb79e26459dae884b158774db1 (diff)
Async scraping, scrape_logs, and CLI
Background scraping: - POST /api/scrape returns 202 immediately with run_id; scraping runs async - GET /api/scrape-status?run_id=xxx polls for results per venue - scrape_logs table: per-venue status (running/ok/error), events_saved, error, timestamps CLI (npm run scrape): - npm run scrape — 全会場をスクレイプ、結果を色付きで出力 - npm run scrape liquid-room — 特定会場のみ - npm run scrape -- --list — 登録済み会場一覧を表示 - エラー時は exit code 1 + エラーメッセージを dim 表示 Venues page: - 最終スクレイプ日時・成否をインラインで表示 - 会場ごとの「更新」ボタンを追加 Bug fix: upsertEvent に description/optional fields のデフォルト値を設定し better-sqlite3 の "Missing named parameter" エラーを解消 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/scrape.ts67
1 files changed, 67 insertions, 0 deletions
diff --git a/scripts/scrape.ts b/scripts/scrape.ts
new file mode 100644
index 0000000..62e9172
--- /dev/null
+++ b/scripts/scrape.ts
@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+/**
+ * CLI スクレイパー
+ *
+ * 使い方:
+ * npm run scrape # 全会場
+ * npm run scrape liquid-room # 特定会場
+ * npm run scrape -- --list # 登録済み会場を表示
+ */
+import { styleText } from "node:util";
+import { runAllScrapers, runScraper } from "../app/lib/scraper-runner.server.js";
+import { ALL_SCRAPERS } from "../app/scrapers/index.js";
+
+const args = process.argv.slice(2);
+const venueId = args.find((a) => !a.startsWith("--"));
+const flagList = args.includes("--list");
+
+if (flagList) {
+ console.log(styleText("bold", "\n登録済みスクレイパー一覧:\n"));
+ for (const s of ALL_SCRAPERS) {
+ console.log(` ${styleText("cyan", s.venue.id.padEnd(25))} ${s.venue.name} (${s.venue.area})`);
+ }
+ console.log();
+ process.exit(0);
+}
+
+const startTime = Date.now();
+
+console.log(
+ styleText("bold", "\n🎸 東京ライブハウス スクレイパー\n") +
+ (venueId
+ ? ` 対象: ${styleText("cyan", venueId)}\n`
+ : ` 対象: ${styleText("cyan", `全 ${ALL_SCRAPERS.length} 会場`)}\n`)
+);
+
+const results = venueId
+ ? [await runScraper(venueId)]
+ : await runAllScrapers();
+
+const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+let totalSaved = 0;
+let hasError = false;
+
+for (const r of results) {
+ const icon = r.status === "ok" ? styleText("green", "✔") : styleText("red", "✖");
+ const name = r.venue_name.padEnd(24);
+
+ if (r.status === "ok") {
+ const count = styleText("green", `${r.events_saved} 件`);
+ console.log(` ${icon} ${name} ${count}`);
+ totalSaved += r.events_saved;
+ } else {
+ hasError = true;
+ console.log(` ${icon} ${styleText("red", name)}`);
+ if (r.error) {
+ const lines = r.error.split("\n");
+ for (const line of lines) {
+ console.log(` ${styleText("dim", line)}`);
+ }
+ }
+ }
+}
+
+const summary = `\n 合計 ${styleText("bold", `${totalSaved} 件`)} 保存 (${elapsed}s)`;
+console.log(summary + "\n");
+
+process.exit(hasError ? 1 : 0);