hetzner-admin: stop/start individual services, shown as STOPPED on status page

- run-command.sh: services / stop <name> / start <name> for app and infra
  stacks (not caddy, hetzner-admin, status). Stopped names go to
  status/stopped.txt (gitignored).
- status: a stopped service that does not respond is STOPPED (gray),
  excluded from uptime and DOWN alerts. If it responds it is UP as usual.
- Infra deploy.sh skips stopped stacks; app repo pushes clear the flag.
- claude-code@.service: Restart=on-failure, and restart-sessions.sh restarts
  one unit at a time, to avoid the tmux "server exited unexpectedly" race.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N5eS3U54gzjRXzm65Np1RS

1 files changed, 42 insertions, 15 deletions

diff --git a/status/app.py b/status/app.py
index 9a20c04..d4b3129 100644
--- a/status/app.py
+++ b/status/app.py
@@ -14,6 +14,8 @@ from html import escape
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 TARGETS_FILE = os.environ.get("TARGETS_FILE", "/app/targets.txt")
+# hetzner-admin で停止したサービス名（1 行 1 つ）。停止中で応答がなければ DOWN ではなく STOPPED 扱い
+STOPPED_FILE = os.path.join(os.path.dirname(TARGETS_FILE), "stopped.txt")
 INTERVAL = int(os.environ.get("CHECK_INTERVAL", "60"))
 TIMEOUT = float(os.environ.get("CHECK_TIMEOUT", "10"))
 HISTORY = int(os.environ.get("HISTORY_SIZE", "90"))
@@ -29,7 +31,7 @@ SUMMARY_HOUR = int(os.environ.get("SUMMARY_HOUR", "8"))
 LOCAL_TZ = timezone(timedelta(hours=float(os.environ.get("SUMMARY_TZ_OFFSET", "9"))))
 
 lock = threading.Lock()
-state = {}  # name -> {"url", "up", "detail", "ms", "checked_at", "history": deque[bool], "fails", "alerted", "day_checks", "day_up"}
+state = {}  # name -> {"url", "up", "stopped", "detail", "ms", "checked_at", "history": deque[bool|None], "fails", "alerted", "day_checks", "day_up"}
 period_start = datetime.now(LOCAL_TZ)  # サマリー集計の開始時刻
 
 
@@ -45,6 +47,14 @@ def load_targets():
     return targets
 
 
+def load_stopped():
+    try:
+        with open(STOPPED_FILE) as f:
+            return {line.strip() for line in f if line.strip()}
+    except FileNotFoundError:
+        return set()
+
+
 def check(url):
     start = time.monotonic()
     parsed = urllib.parse.urlparse(url)
@@ -86,6 +96,7 @@ def notify(title, message, tags, priority="default"):
 
 def run_checks():
     targets = load_targets()
+    stopped_names = load_stopped()
     with ThreadPoolExecutor(max_workers=max(1, len(targets))) as pool:
         results = list(pool.map(lambda t: check(t[1]), targets))
     now = datetime.now(timezone.utc).isoformat(timespec="seconds")
@@ -99,7 +110,16 @@ def run_checks():
                 {"history": deque(maxlen=HISTORY), "fails": 0, "alerted": False,
                  "day_checks": 0, "day_up": 0},
             )
-            entry.update(url=url, up=up, detail=detail, ms=ms, checked_at=now)
+            # 停止中でも応答があれば（push で再デプロイされた等）普通に UP として扱う
+            stopped = name in stopped_names and not up
+            if stopped:
+                detail = "Stopped from admin"
+            entry.update(url=url, up=up, stopped=stopped, detail=detail, ms=ms, checked_at=now)
+            if stopped:
+                # 稼働率・DOWN 通知の対象外。履歴は灰色で残す
+                entry["history"].append(None)
+                entry["fails"] = 0
+                continue
             entry["history"].append(up)
             entry["day_checks"] += 1
             entry["day_up"] += up
@@ -118,7 +138,7 @@ def send_summary(now):
     global period_start
     with lock:
         rows = [
-            (100 * e["day_up"] / e["day_checks"], name, e["up"])
+            (100 * e["day_up"] / e["day_checks"], name, e["up"] or e["stopped"])
             for name, e in state.items()
             if e["day_checks"]
         ]
@@ -158,37 +178,44 @@ def checker_loop():
 
 def snapshot():
     with lock:
-        services = [
-            {
+        services = []
+        for name, e in state.items():
+            checked = [h for h in e["history"] if h is not None]
+            services.append({
                 "name": name,
                 "url": e["url"],
                 "up": e["up"],
+                "stopped": e["stopped"],
+                "state": "stopped" if e["stopped"] else "up" if e["up"] else "down",
                 "detail": e["detail"],
                 "ms": e["ms"],
                 "checked_at": e["checked_at"],
-                "uptime": round(100 * sum(e["history"]) / len(e["history"]), 1),
+                "uptime": round(100 * sum(checked) / len(checked), 1) if checked else None,
                 "history": list(e["history"]),
-            }
-            for name, e in state.items()
-        ]
-    return {"all_up": all(s["up"] for s in services), "services": services}
+            })
+    return {"all_up": all(s["up"] or s["stopped"] for s in services), "services": services}
 
 
 def render_html(data):
     rows = []
     for s in data["services"]:
         bars = "".join(
-            f'<i class="{"u" if ok else "d"}"></i>' for ok in s["history"]
+            f'<i class="{"s" if ok is None else "u" if ok else "d"}"></i>' for ok in s["history"]
         )
+        dot = {"up": "u", "down": "d", "stopped": "s"}[s["state"]]
+        uptime = "-" if s["uptime"] is None else f'{s["uptime"]}%'
+        ms = "-" if s["stopped"] else f'{s["ms"]} ms'
         rows.append(
-            f'<tr><td><span class="dot {"u" if s["up"] else "d"}"></span>{escape(s["name"])}</td>'
-            f'<td>{escape(s["detail"])}</td><td>{s["ms"]} ms</td><td>{s["uptime"]}%</td>'
+            f'<tr><td><span class="dot {dot}"></span>{escape(s["name"])}</td>'
+            f'<td>{escape(s["detail"])}</td><td>{ms}</td><td>{uptime}</td>'
             f'<td class="bars">{bars}</td></tr>'
         )
     if not data["services"]:
         banner = '<div class="banner">Checking...</div>'
     elif data["all_up"]:
-        banner = '<div class="banner u">All systems operational</div>'
+        n_stopped = sum(s["stopped"] for s in data["services"])
+        note = f" ({n_stopped} stopped)" if n_stopped else ""
+        banner = f'<div class="banner u">All systems operational{note}</div>'
     else:
         banner = '<div class="banner d">Some systems are down</div>'
     checked = data["services"][0]["checked_at"] if data["services"] else "-"
@@ -204,7 +231,7 @@ body{{font-family:system-ui,sans-serif;max-width:860px;margin:2em auto;padding:0
 .banner.u{{background:#2a9d4b}}.banner.d{{background:#d0393e}}
 table{{width:100%;border-collapse:collapse;margin-top:1em}}td{{padding:.5em;border-bottom:1px solid #eee;white-space:nowrap}}
 .dot{{display:inline-block;width:10px;height:10px;border-radius:50%;margin-right:.5em}}
-.u{{background:#2a9d4b}}.d{{background:#d0393e}}
+.u{{background:#2a9d4b}}.d{{background:#d0393e}}.s{{background:#bbb}}
 .bars{{width:100%}}.bars i{{display:inline-block;width:4px;height:18px;margin-right:1px;border-radius:1px}}
 small{{color:#888}}
 </style></head><body>