summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoryyamashita <yyamashita@mosquit.one>2026-05-06 22:24:38 +0900
committeryyamashita <yyamashita@mosquit.one>2026-05-06 22:24:38 +0900
commit538fd636e25595d88a958344d285c0e7cf44e530 (patch)
treeeb2999f355570224fa96877d5043af2ef3ec76ef
parentf817604858891edb79e26459dae884b158774db1 (diff)
Async scraping, scrape_logs, and CLI
Background scraping: - POST /api/scrape returns 202 immediately with run_id; scraping runs async - GET /api/scrape-status?run_id=xxx polls for results per venue - scrape_logs table: per-venue status (running/ok/error), events_saved, error, timestamps CLI (npm run scrape): - npm run scrape — 全会場をスクレイプ、結果を色付きで出力 - npm run scrape liquid-room — 特定会場のみ - npm run scrape -- --list — 登録済み会場一覧を表示 - エラー時は exit code 1 + エラーメッセージを dim 表示 Venues page: - 最終スクレイプ日時・成否をインラインで表示 - 会場ごとの「更新」ボタンを追加 Bug fix: upsertEvent に description/optional fields のデフォルト値を設定し better-sqlite3 の "Missing named parameter" エラーを解消 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
-rw-r--r--app/lib/db.server.ts102
-rw-r--r--app/lib/scraper-runner.server.ts81
-rw-r--r--app/routes.ts1
-rw-r--r--app/routes/api.scrape-status.ts16
-rw-r--r--app/routes/api.scrape.ts34
-rw-r--r--app/routes/venues.tsx115
-rw-r--r--package-lock.json528
-rw-r--r--package.json4
-rw-r--r--scripts/scrape.ts67
9 files changed, 861 insertions, 87 deletions
diff --git a/app/lib/db.server.ts b/app/lib/db.server.ts
index 0c55991..26735c6 100644
--- a/app/lib/db.server.ts
+++ b/app/lib/db.server.ts
@@ -45,6 +45,21 @@ function initSchema(db: Database.Database) {
CREATE INDEX IF NOT EXISTS idx_events_date ON events(date);
CREATE INDEX IF NOT EXISTS idx_events_venue_id ON events(venue_id);
+
+ CREATE TABLE IF NOT EXISTS scrape_logs (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ run_id TEXT NOT NULL,
+ venue_id TEXT NOT NULL,
+ venue_name TEXT NOT NULL,
+ status TEXT NOT NULL DEFAULT 'running', -- running | ok | error
+ events_saved INTEGER NOT NULL DEFAULT 0,
+ error TEXT,
+ started_at TEXT NOT NULL DEFAULT (datetime('now')),
+ finished_at TEXT
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_scrape_logs_run_id ON scrape_logs(run_id);
+ CREATE INDEX IF NOT EXISTS idx_scrape_logs_venue_id ON scrape_logs(venue_id);
`);
}
@@ -102,7 +117,19 @@ export function upsertVenue(
.run(id, name, url, area ?? null);
}
-export function upsertEvent(event: EventInput) {
+export function upsertEvent(raw: EventInput) {
+ // Ensure all named parameters exist (better-sqlite3 requires them all)
+ const event = {
+ artist: null,
+ start_time: null,
+ open_time: null,
+ ticket_url: null,
+ price: null,
+ image_url: null,
+ description: null,
+ source_url: null,
+ ...raw,
+ };
getDb()
.prepare(
`INSERT INTO events
@@ -190,3 +217,76 @@ export function getVenues(): Venue[] {
)
.all() as Venue[];
}
+
+// ---------- Scrape logs ----------
+
+export interface ScrapeLog {
+ id: number;
+ run_id: string;
+ venue_id: string;
+ venue_name: string;
+ status: "running" | "ok" | "error";
+ events_saved: number;
+ error: string | null;
+ started_at: string;
+ finished_at: string | null;
+}
+
+export function insertScrapeLog(
+ run_id: string,
+ venue_id: string,
+ venue_name: string
+): number {
+ const result = getDb()
+ .prepare(
+ `INSERT INTO scrape_logs (run_id, venue_id, venue_name, status)
+ VALUES (?, ?, ?, 'running')`
+ )
+ .run(run_id, venue_id, venue_name);
+ return result.lastInsertRowid as number;
+}
+
+export function updateScrapeLog(
+ id: number,
+ status: "ok" | "error",
+ events_saved: number,
+ error?: string
+) {
+ getDb()
+ .prepare(
+ `UPDATE scrape_logs
+ SET status = ?, events_saved = ?, error = ?, finished_at = datetime('now')
+ WHERE id = ?`
+ )
+ .run(status, events_saved, error ?? null, id);
+}
+
+export function getLatestScrapeRun(): ScrapeLog[] {
+ return getDb()
+ .prepare(
+ `SELECT * FROM scrape_logs
+ WHERE run_id = (SELECT run_id FROM scrape_logs ORDER BY started_at DESC LIMIT 1)
+ ORDER BY id ASC`
+ )
+ .all() as ScrapeLog[];
+}
+
+export function getScrapeRunById(run_id: string): ScrapeLog[] {
+ return getDb()
+ .prepare(
+ "SELECT * FROM scrape_logs WHERE run_id = ? ORDER BY id ASC"
+ )
+ .all(run_id) as ScrapeLog[];
+}
+
+export function getLastScrapePerVenue(): ScrapeLog[] {
+ return getDb()
+ .prepare(
+ `SELECT s.* FROM scrape_logs s
+ INNER JOIN (
+ SELECT venue_id, MAX(started_at) AS latest FROM scrape_logs GROUP BY venue_id
+ ) t ON s.venue_id = t.venue_id AND s.started_at = t.latest
+ ORDER BY s.venue_name ASC`
+ )
+ .all() as ScrapeLog[];
+}
diff --git a/app/lib/scraper-runner.server.ts b/app/lib/scraper-runner.server.ts
index 191dd00..87dd16c 100644
--- a/app/lib/scraper-runner.server.ts
+++ b/app/lib/scraper-runner.server.ts
@@ -1,9 +1,16 @@
-import { upsertVenue, upsertEvent } from "./db.server";
+import { randomUUID } from "crypto";
+import {
+ upsertVenue,
+ upsertEvent,
+ insertScrapeLog,
+ updateScrapeLog,
+ type ScrapeLog,
+} from "./db.server";
import { generateVenueMarkdown, generateAllVenueMarkdown } from "./markdown-writer.server";
import { ALL_SCRAPERS } from "~/scrapers/index";
import type { EventInput } from "./db.server";
-const SCRAPE_WINDOW_DAYS = 35; // ~1 month
+const SCRAPE_WINDOW_DAYS = 35;
function scrapeWindow(): { from: string; to: string } {
const from = new Date();
@@ -21,81 +28,81 @@ function withinWindow(event: EventInput, from: string, to: string): boolean {
}
export interface ScrapeResult {
+ run_id: string;
venue_id: string;
venue_name: string;
+ status: "ok" | "error";
events_saved: number;
- markdown_path?: string;
error?: string;
}
-export async function runAllScrapers(): Promise<ScrapeResult[]> {
+/** Fire-and-forget: start all scrapers in the background, return run_id immediately. */
+export function startAllScrapersAsync(): string {
+ const run_id = randomUUID();
+ // Don't await — runs in background
+ void runAllScrapers(run_id);
+ return run_id;
+}
+
+export function startScraperAsync(venueId: string): string {
+ const run_id = randomUUID();
+ void runScraper(venueId, run_id);
+ return run_id;
+}
+
+/** Runs all scrapers, writes logs to DB. Can be awaited (e.g. from CLI). */
+export async function runAllScrapers(run_id = randomUUID()): Promise<ScrapeResult[]> {
const results: ScrapeResult[] = [];
const successIds: string[] = [];
for (const scraper of ALL_SCRAPERS) {
const { venue } = scraper;
upsertVenue(venue.id, venue.name, venue.url, venue.area);
+ const logId = insertScrapeLog(run_id, venue.id, venue.name);
try {
const { from, to } = scrapeWindow();
- const events = (await scraper.scrape()).filter((e) =>
- withinWindow(e, from, to)
- );
+ const events = (await scraper.scrape()).filter((e) => withinWindow(e, from, to));
for (const event of events) {
upsertEvent(event);
}
+ updateScrapeLog(logId, "ok", events.length);
successIds.push(venue.id);
- results.push({
- venue_id: venue.id,
- venue_name: venue.name,
- events_saved: events.length,
- });
+ results.push({ run_id, venue_id: venue.id, venue_name: venue.name, status: "ok", events_saved: events.length });
} catch (err) {
- results.push({
- venue_id: venue.id,
- venue_name: venue.name,
- events_saved: 0,
- error: err instanceof Error ? err.message : String(err),
- });
+ const error = err instanceof Error ? err.message : String(err);
+ updateScrapeLog(logId, "error", 0, error);
+ results.push({ run_id, venue_id: venue.id, venue_name: venue.name, status: "error", events_saved: 0, error });
}
}
- // Generate Markdown files for all venues that scraped successfully
generateAllVenueMarkdown(successIds);
-
return results;
}
-export async function runScraper(venueId: string): Promise<ScrapeResult> {
+/** Runs a single scraper by venue ID. */
+export async function runScraper(venueId: string, run_id = randomUUID()): Promise<ScrapeResult> {
const scraper = ALL_SCRAPERS.find((s) => s.venue.id === venueId);
if (!scraper) {
- return { venue_id: venueId, venue_name: venueId, events_saved: 0, error: "Scraper not found" };
+ return { run_id, venue_id: venueId, venue_name: venueId, status: "error", events_saved: 0, error: "Scraper not found" };
}
const { venue } = scraper;
upsertVenue(venue.id, venue.name, venue.url, venue.area);
+ const logId = insertScrapeLog(run_id, venue.id, venue.name);
try {
const { from, to } = scrapeWindow();
- const events = (await scraper.scrape()).filter((e) =>
- withinWindow(e, from, to)
- );
+ const events = (await scraper.scrape()).filter((e) => withinWindow(e, from, to));
for (const event of events) {
upsertEvent(event);
}
+ updateScrapeLog(logId, "ok", events.length);
generateVenueMarkdown(venue.id);
- return {
- venue_id: venue.id,
- venue_name: venue.name,
- events_saved: events.length,
- markdown_path: `events/${venue.id}.md`,
- };
+ return { run_id, venue_id: venue.id, venue_name: venue.name, status: "ok", events_saved: events.length };
} catch (err) {
- return {
- venue_id: venue.id,
- venue_name: venue.name,
- events_saved: 0,
- error: err instanceof Error ? err.message : String(err),
- };
+ const error = err instanceof Error ? err.message : String(err);
+ updateScrapeLog(logId, "error", 0, error);
+ return { run_id, venue_id: venue.id, venue_name: venue.name, status: "error", events_saved: 0, error };
}
}
diff --git a/app/routes.ts b/app/routes.ts
index 028da16..c0096e1 100644
--- a/app/routes.ts
+++ b/app/routes.ts
@@ -8,4 +8,5 @@ export default [
]),
route("venues", "routes/venues.tsx"),
route("api/scrape", "routes/api.scrape.ts"),
+ route("api/scrape-status", "routes/api.scrape-status.ts"),
] satisfies RouteConfig;
diff --git a/app/routes/api.scrape-status.ts b/app/routes/api.scrape-status.ts
new file mode 100644
index 0000000..28d08d4
--- /dev/null
+++ b/app/routes/api.scrape-status.ts
@@ -0,0 +1,16 @@
+/**
+ * GET /api/scrape-status?run_id=xxx — 指定 run_id の結果を返す
+ * GET /api/scrape-status — 最新 run の結果を返す
+ */
+import type { Route } from "./+types/api.scrape-status";
+import { getScrapeRunById, getLatestScrapeRun } from "~/lib/db.server";
+
+export async function loader({ request }: Route.LoaderArgs) {
+ const url = new URL(request.url);
+ const run_id = url.searchParams.get("run_id");
+
+ const logs = run_id ? getScrapeRunById(run_id) : getLatestScrapeRun();
+ const running = logs.some((l) => l.status === "running");
+
+ return Response.json({ running, results: logs });
+}
diff --git a/app/routes/api.scrape.ts b/app/routes/api.scrape.ts
index 4071985..f9daa5c 100644
--- a/app/routes/api.scrape.ts
+++ b/app/routes/api.scrape.ts
@@ -1,37 +1,37 @@
/**
- * Resource route: POST /api/scrape
- * Triggers scraping for all venues (or a specific one via ?venue_id=xxx).
- * Returns JSON results and redirects back if called from a form.
+ * Resource route: /api/scrape
+ *
+ * POST (form action) — バックグラウンドでスクレイプ開始、202 を即時返却
+ * GET ?venue_id=xxx — 特定会場のみバックグラウンド開始
+ * GET (パラメータなし) — 全会場をバックグラウンド開始
+ *
+ * ステータス確認は /api/scrape-status?run_id=xxx
*/
import { redirect } from "react-router";
import type { Route } from "./+types/api.scrape";
-import { runAllScrapers, runScraper } from "~/lib/scraper-runner.server";
+import { startAllScrapersAsync, startScraperAsync } from "~/lib/scraper-runner.server";
export async function action({ request }: Route.ActionArgs) {
const formData = await request.formData();
const venueId = formData.get("venue_id");
- const results = venueId
- ? [await runScraper(String(venueId))]
- : await runAllScrapers();
+ const run_id = venueId
+ ? startScraperAsync(String(venueId))
+ : startAllScrapersAsync();
- // If called from a browser form, redirect back
const referer = request.headers.get("Referer");
- if (referer) {
- return redirect(referer);
- }
+ if (referer) return redirect(referer);
- return Response.json({ results });
+ return Response.json({ run_id, status: "started" }, { status: 202 });
}
-// Allow GET for quick testing in the browser
export async function loader({ request }: Route.LoaderArgs) {
const url = new URL(request.url);
const venueId = url.searchParams.get("venue_id");
- const results = venueId
- ? [await runScraper(venueId)]
- : await runAllScrapers();
+ const run_id = venueId
+ ? startScraperAsync(venueId)
+ : startAllScrapersAsync();
- return Response.json({ results });
+ return Response.json({ run_id, status: "started" }, { status: 202 });
}
diff --git a/app/routes/venues.tsx b/app/routes/venues.tsx
index 23b052f..affa72a 100644
--- a/app/routes/venues.tsx
+++ b/app/routes/venues.tsx
@@ -1,17 +1,19 @@
-import { useLoaderData, Link } from "react-router";
+import { useLoaderData, Link, Form } from "react-router";
import type { Route } from "./+types/venues";
-import { getVenues } from "~/lib/db.server";
+import { getVenues, getLastScrapePerVenue, type ScrapeLog } from "~/lib/db.server";
import { getScraperIds } from "~/lib/venue-meta.server";
export async function loader(_: Route.LoaderArgs) {
const venues = getVenues();
const scraperIds = getScraperIds();
- return { venues, scraperIds };
+ const scrapeStatus = getLastScrapePerVenue();
+ return { venues, scraperIds, scrapeStatus };
}
export default function Venues() {
- const { venues, scraperIds: scraperIdList } = useLoaderData<typeof loader>();
+ const { venues, scraperIds: scraperIdList, scrapeStatus } = useLoaderData<typeof loader>();
const scraperIds = new Set(scraperIdList);
+ const statusMap = new Map<string, ScrapeLog>(scrapeStatus.map((s) => [s.venue_id, s]));
return (
<div className="min-h-screen bg-gray-950 text-gray-100">
@@ -26,43 +28,94 @@ export default function Venues() {
</header>
<main className="max-w-4xl mx-auto px-4 py-10">
- <div className="mb-8">
- <h1 className="text-2xl font-bold">会場一覧</h1>
- <p className="mt-1 text-sm text-gray-400">
- 現在 {scraperIdList.length} 会場のスクレイパーが登録されています。
- 新しい会場を追加するには <code className="bg-gray-800 px-1 rounded">app/scrapers/</code> に
- モジュールを追加して <code className="bg-gray-800 px-1 rounded">index.ts</code> に登録してください。
- </p>
+ <div className="mb-8 flex items-start justify-between gap-4 flex-wrap">
+ <div>
+ <h1 className="text-2xl font-bold">会場一覧</h1>
+ <p className="mt-1 text-sm text-gray-400">
+ 現在 {scraperIdList.length} 会場のスクレイパーが登録されています。
+ </p>
+ </div>
+ <Form method="post" action="/api/scrape">
+ <button
+ type="submit"
+ className="rounded-md bg-indigo-600 px-4 py-2 text-sm font-medium hover:bg-indigo-500 transition-colors"
+ >
+ 全会場を更新
+ </button>
+ </Form>
</div>
{venues.length === 0 ? (
- <p className="text-gray-500">まだ会場データがありません。「情報を更新」してください。</p>
+ <p className="text-gray-500">まだ会場データがありません。「全会場を更新」してください。</p>
) : (
- <div className="grid gap-4 sm:grid-cols-2">
- {venues.map((v) => (
- <Link
- key={v.id}
- to={`/events?venue_id=${v.id}`}
- className="flex items-center justify-between rounded-xl bg-gray-800/60 p-4 hover:bg-gray-800 transition-colors border border-gray-700/50"
- >
- <div>
- <p className="font-semibold">{v.name}</p>
- {v.area && <p className="text-sm text-gray-400">{v.area}</p>}
+ <div className="grid gap-3">
+ {venues.map((v) => {
+ const log = statusMap.get(v.id);
+ return (
+ <div
+ key={v.id}
+ className="flex items-center gap-4 rounded-xl bg-gray-800/60 border border-gray-700/40 p-4"
+ >
+ {/* 会場名 + エリア */}
+ <div className="flex-1 min-w-0">
+ <Link
+ to={`/events?venue_id=${v.id}`}
+ className="font-semibold hover:text-indigo-300 transition-colors"
+ >
+ {v.name}
+ </Link>
+ {v.area && <p className="text-xs text-gray-400">{v.area}</p>}
+ </div>
+
+ {/* イベント件数 */}
+ <span className="text-sm text-gray-400 whitespace-nowrap">
+ <span className="text-lg font-bold text-gray-200">{v.event_count ?? 0}</span> 件
+ </span>
+
+ {/* 最終スクレイプ状態 */}
+ {log ? (
+ <ScrapeStatus log={log} />
+ ) : (
+ <span className="text-xs text-gray-600 whitespace-nowrap">未実行</span>
+ )}
+
+ {/* 個別更新ボタン */}
{scraperIds.has(v.id) && (
- <span className="mt-1 inline-block rounded-full bg-emerald-700/40 px-2 py-0.5 text-xs text-emerald-300">
- スクレイパー登録済
- </span>
+ <Form method="post" action="/api/scrape">
+ <input type="hidden" name="venue_id" value={v.id} />
+ <button
+ type="submit"
+ className="rounded bg-gray-700 px-3 py-1 text-xs hover:bg-gray-600 transition-colors whitespace-nowrap"
+ >
+ 更新
+ </button>
+ </Form>
)}
</div>
- <span className="text-2xl font-bold text-gray-500">
- {v.event_count ?? 0}
- <span className="text-sm font-normal ml-1">件</span>
- </span>
- </Link>
- ))}
+ );
+ })}
</div>
)}
</main>
</div>
);
}
+
+function ScrapeStatus({ log }: { log: ScrapeLog }) {
+ if (log.status === "running") {
+ return <span className="text-xs text-yellow-400 whitespace-nowrap">⟳ 実行中...</span>;
+ }
+ if (log.status === "error") {
+ return (
+ <span className="text-xs text-red-400 whitespace-nowrap" title={log.error ?? ""}>
+ ✖ エラー
+ </span>
+ );
+ }
+ const time = log.finished_at?.slice(0, 16).replace("T", " ") ?? "";
+ return (
+ <span className="text-xs text-emerald-400 whitespace-nowrap" title={time}>
+ ✔ {time}
+ </span>
+ );
+}
diff --git a/package-lock.json b/package-lock.json
index daa9bae..95db447 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -23,6 +23,7 @@
"@types/react": "^19.2.14",
"@types/react-dom": "^19.2.3",
"tailwindcss": "^4.2.4",
+ "tsx": "^4.20.3",
"typescript": "^5.9.3",
"vite": "^6.3.5",
"vite-tsconfig-paths": "^6.1.1"
@@ -2994,6 +2995,19 @@
"node": ">= 0.4"
}
},
+ "node_modules/get-tsconfig": {
+ "version": "4.14.0",
+ "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
+ "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "resolve-pkg-maps": "^1.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
+ }
+ },
"node_modules/github-from-package": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
@@ -4270,6 +4284,16 @@
"url": "https://paulmillr.com/funding/"
}
},
+ "node_modules/resolve-pkg-maps": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
+ "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
+ "dev": true,
+ "license": "MIT",
+ "funding": {
+ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
+ }
+ },
"node_modules/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
@@ -4847,6 +4871,510 @@
"dev": true,
"optional": true
},
+ "node_modules/tsx": {
+ "version": "4.21.0",
+ "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz",
+ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "esbuild": "~0.27.0",
+ "get-tsconfig": "^4.7.5"
+ },
+ "bin": {
+ "tsx": "dist/cli.mjs"
+ },
+ "engines": {
+ "node": ">=18.0.0"
+ },
+ "optionalDependencies": {
+ "fsevents": "~2.3.3"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/aix-ppc64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz",
+ "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==",
+ "cpu": [
+ "ppc64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "aix"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/android-arm": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz",
+ "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==",
+ "cpu": [
+ "arm"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/android-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz",
+ "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/android-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz",
+ "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/darwin-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz",
+ "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/darwin-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz",
+ "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz",
+ "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "freebsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/freebsd-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz",
+ "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "freebsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-arm": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz",
+ "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==",
+ "cpu": [
+ "arm"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz",
+ "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-ia32": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz",
+ "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==",
+ "cpu": [
+ "ia32"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-loong64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz",
+ "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==",
+ "cpu": [
+ "loong64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-mips64el": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz",
+ "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==",
+ "cpu": [
+ "mips64el"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-ppc64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz",
+ "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==",
+ "cpu": [
+ "ppc64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-riscv64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz",
+ "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==",
+ "cpu": [
+ "riscv64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-s390x": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz",
+ "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==",
+ "cpu": [
+ "s390x"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/linux-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz",
+ "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz",
+ "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "netbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/netbsd-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz",
+ "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "netbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz",
+ "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "openbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/openbsd-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz",
+ "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "openbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz",
+ "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "openharmony"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/sunos-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz",
+ "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "sunos"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/win32-arm64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz",
+ "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/win32-ia32": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz",
+ "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==",
+ "cpu": [
+ "ia32"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/@esbuild/win32-x64": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz",
+ "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/tsx/node_modules/esbuild": {
+ "version": "0.27.7",
+ "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz",
+ "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==",
+ "dev": true,
+ "hasInstallScript": true,
+ "license": "MIT",
+ "bin": {
+ "esbuild": "bin/esbuild"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "optionalDependencies": {
+ "@esbuild/aix-ppc64": "0.27.7",
+ "@esbuild/android-arm": "0.27.7",
+ "@esbuild/android-arm64": "0.27.7",
+ "@esbuild/android-x64": "0.27.7",
+ "@esbuild/darwin-arm64": "0.27.7",
+ "@esbuild/darwin-x64": "0.27.7",
+ "@esbuild/freebsd-arm64": "0.27.7",
+ "@esbuild/freebsd-x64": "0.27.7",
+ "@esbuild/linux-arm": "0.27.7",
+ "@esbuild/linux-arm64": "0.27.7",
+ "@esbuild/linux-ia32": "0.27.7",
+ "@esbuild/linux-loong64": "0.27.7",
+ "@esbuild/linux-mips64el": "0.27.7",
+ "@esbuild/linux-ppc64": "0.27.7",
+ "@esbuild/linux-riscv64": "0.27.7",
+ "@esbuild/linux-s390x": "0.27.7",
+ "@esbuild/linux-x64": "0.27.7",
+ "@esbuild/netbsd-arm64": "0.27.7",
+ "@esbuild/netbsd-x64": "0.27.7",
+ "@esbuild/openbsd-arm64": "0.27.7",
+ "@esbuild/openbsd-x64": "0.27.7",
+ "@esbuild/openharmony-arm64": "0.27.7",
+ "@esbuild/sunos-x64": "0.27.7",
+ "@esbuild/win32-arm64": "0.27.7",
+ "@esbuild/win32-ia32": "0.27.7",
+ "@esbuild/win32-x64": "0.27.7"
+ }
+ },
"node_modules/tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
diff --git a/package.json b/package.json
index 16f2708..823b578 100644
--- a/package.json
+++ b/package.json
@@ -6,7 +6,8 @@
"build": "react-router build",
"dev": "react-router dev",
"start": "react-router-serve ./build/server/index.js",
- "typecheck": "react-router typegen && tsc"
+ "typecheck": "react-router typegen && tsc",
+ "scrape": "node --import tsx/esm scripts/scrape.ts"
},
"dependencies": {
"@react-router/node": "^7.3.0",
@@ -19,6 +20,7 @@
"react-router": "^7.3.0"
},
"devDependencies": {
+ "tsx": "^4.20.3",
"@react-router/dev": "^7.3.0",
"@tailwindcss/vite": "^4.2.4",
"@types/better-sqlite3": "^7.6.13",
diff --git a/scripts/scrape.ts b/scripts/scrape.ts
new file mode 100644
index 0000000..62e9172
--- /dev/null
+++ b/scripts/scrape.ts
@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+/**
+ * CLI スクレイパー
+ *
+ * 使い方:
+ * npm run scrape # 全会場
+ * npm run scrape liquid-room # 特定会場
+ * npm run scrape -- --list # 登録済み会場を表示
+ */
+import { styleText } from "node:util";
+import { runAllScrapers, runScraper } from "../app/lib/scraper-runner.server.js";
+import { ALL_SCRAPERS } from "../app/scrapers/index.js";
+
+const args = process.argv.slice(2);
+const venueId = args.find((a) => !a.startsWith("--"));
+const flagList = args.includes("--list");
+
+if (flagList) {
+ console.log(styleText("bold", "\n登録済みスクレイパー一覧:\n"));
+ for (const s of ALL_SCRAPERS) {
+ console.log(` ${styleText("cyan", s.venue.id.padEnd(25))} ${s.venue.name} (${s.venue.area})`);
+ }
+ console.log();
+ process.exit(0);
+}
+
+const startTime = Date.now();
+
+console.log(
+ styleText("bold", "\n🎸 東京ライブハウス スクレイパー\n") +
+ (venueId
+ ? ` 対象: ${styleText("cyan", venueId)}\n`
+ : ` 対象: ${styleText("cyan", `全 ${ALL_SCRAPERS.length} 会場`)}\n`)
+);
+
+const results = venueId
+ ? [await runScraper(venueId)]
+ : await runAllScrapers();
+
+const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+let totalSaved = 0;
+let hasError = false;
+
+for (const r of results) {
+ const icon = r.status === "ok" ? styleText("green", "✔") : styleText("red", "✖");
+ const name = r.venue_name.padEnd(24);
+
+ if (r.status === "ok") {
+ const count = styleText("green", `${r.events_saved} 件`);
+ console.log(` ${icon} ${name} ${count}`);
+ totalSaved += r.events_saved;
+ } else {
+ hasError = true;
+ console.log(` ${icon} ${styleText("red", name)}`);
+ if (r.error) {
+ const lines = r.error.split("\n");
+ for (const line of lines) {
+ console.log(` ${styleText("dim", line)}`);
+ }
+ }
+ }
+}
+
+const summary = `\n 合計 ${styleText("bold", `${totalSaved} 件`)} 保存 (${elapsed}s)`;
+console.log(summary + "\n");
+
+process.exit(hasError ? 1 : 0);