Recipe: status monitoring

#!/usr/bin/env bash
# Exits 1 if any chain is not ok. Use it from cron or a health check.
bad=$(curl -sf --max-time 10 "https://farbeamlabs.com/api/v1/status" \
  | jq -r '.data.chains[] | select(.status != "ok") | "\(.chain) \(.status) \(.last_error // "")"') || { echo "status endpoint failed"; exit 1; }
if [ -n "$bad" ]; then echo "$bad"; exit 1; fi
echo ok

// monitor.mjs: run with node monitor.mjs
const STATUS = "https://farbeamlabs.com/api/v1/status";
const INTERVAL_MS = 60000;
const ALERT_AFTER = 3; // bad checks in a row
const strikes = new Map();
let lastUptime = null;

async function alert(message) {
  // Replace this with a call to your own pager or chat.
  console.error("ALERT", new Date().toISOString(), message);
}

async function check() {
  const res = await fetch(STATUS, { signal: AbortSignal.timeout(10000) });
  if (!res.ok) throw new Error(`HTTP ${res.status}`);
  const { data } = await res.json();
  if (lastUptime !== null && data.uptime_seconds < lastUptime) await alert("service restarted, expect a gap in rows");
  lastUptime = data.uptime_seconds;
  for (const chain of data.chains) {
    const count = chain.status === "ok" ? 0 : (strikes.get(chain.chain) ?? 0) + 1;
    strikes.set(chain.chain, count);
    if (count === ALERT_AFTER) await alert(`${chain.chain} is ${chain.status}: ${chain.last_error ?? "no error text"}`);
  }
  for (const source of data.sources.filter((s) => s.ok === false)) {
    console.log("source failing", source.name, source.error);
  }
}

setInterval(() => check().catch((err) => alert(`status check failed: ${err.message}`)), INTERVAL_MS);
await check();

# monitor.py: run with python3 monitor.py
import time
from datetime import datetime, timezone
import requests

STATUS = "https://farbeamlabs.com/api/v1/status"
ALERT_AFTER = 3  # bad checks in a row
strikes, last_uptime = {}, None

def alert(message):
    # Replace this with a call to your own pager or chat.
    print("ALERT", datetime.now(timezone.utc).isoformat(), message)

while True:
    try:
        res = requests.get(STATUS, timeout=10)
        res.raise_for_status()
        data = res.json()["data"]
        if last_uptime is not None and data["uptime_seconds"] < last_uptime:
            alert("service restarted, expect a gap in rows")
        last_uptime = data["uptime_seconds"]
        for chain in data["chains"]:
            count = 0 if chain["status"] == "ok" else strikes.get(chain["chain"], 0) + 1
            strikes[chain["chain"]] = count
            if count == ALERT_AFTER:
                alert(f"{chain['chain']} is {chain['status']}: {chain['last_error']}")
        for source in data["sources"]:
            if source["ok"] is False:
                print("source failing", source["name"], source["error"])
    except (requests.RequestException, ValueError) as err:
        alert(f"status check failed: {err}")
    time.sleep(60)

curl -s "https://farbeamlabs.com/api/v1/rules" -H "Authorization: Bearer $FARBEAM_KEY" \
  | jq -r '.data[] | select(.enabled == false or .consecutive_failures >= 3) | [.id, .name, .enabled, .consecutive_failures] | @tsv'

const res = await fetch("https://farbeamlabs.com/api/v1/rules", {
  headers: { Authorization: `Bearer ${process.env.FARBEAM_KEY}` },
});
const { data } = await res.json();
for (const rule of data) {
  if (!rule.enabled || rule.consecutive_failures >= 3) console.log(rule.id, rule.name, rule.enabled, rule.consecutive_failures);
}

import os, requests

res = requests.get(
    "https://farbeamlabs.com/api/v1/rules",
    headers={"Authorization": f"Bearer {os.environ['FARBEAM_KEY']}"},
    timeout=30,
)
for rule in res.json()["data"]:
    if not rule["enabled"] or rule["consecutive_failures"] >= 3:
        print(rule["id"], rule["name"], rule["enabled"], rule["consecutive_failures"])