Compare commits

8 Commits

Author SHA1 Message Date
myron ee0e116963 Add copyright notice 2026-07-26 23:53:37 -05:00
myron adbd1a7a24 Stop flagging parkerslingshotrentals as DOWN: it's intentionally behind a Basic Auth Coming Soon gate during rebuild, so 401 is expected there, not a failure
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-19 00:51:54 -05:00
myron 73aac8ab01 Fix weather widget location label stuck on Fort Worth: the span was a static HTML placeholder never wired to the API response. Wire loadWeather() to update it from d.location, and correct the fallback text to Weatherford, TX
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-19 00:45:04 -05:00
myron 646da21b86 Update weather section location to Weatherford, TX 76088 (from Fort Worth)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-18 19:55:05 -05:00
myron d97a345672 Fix network map duplicate-device bug: drop nmap --send-ip (was causing MAC misattribution/stale ARP entries) and dedupe network_devices by MAC on every scan push so IP changes don't leave orphaned rows
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-18 19:53:17 -05:00
Claude 141191bcdd Fix jarvis-health.sh: watchdog-restart alerts never deduped or auto-resolved
source_key was minute-stamped (health:wd_restart:YYYYMMDDHHMM), so the same restart event seen across two 5-min cron runs within the logs 6-min lookback window got two different keys -> two alert rows + two emails, and the key never matched a clear_cond call so these rows stayed resolved=0 forever, accumulating in the active-alerts view.

Fixed to a stable key (health:wd_restart), matching the pattern used by the other three checks in this script: raise() now dedups via the existing COUNT..resolved=0 check, and clear_cond() runs when no recent restart line is found.

Verified live: injected a fake watchdog restart log line, ran the script twice -> exactly 1 alert row + 1 email (not 2). Removed the line -> alert auto-resolved (resolved=1).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-08 07:43:20 -05:00
Claude 8399048252 Backup: capture Phase 1/2 env files, systemd drop-ins, and ops scripts
jarvis-backup.sh now also archives /etc/jarvis-arc (reactor.env), /etc/jarvis (db.env), the jarvis-arc.service.d systemd drop-ins, and /usr/local/bin/jarvis-*.sh. Previously a restore would have come back with no API keys or DB password since those moved outside /var/www/jarvis + /opt/jarvis-arc during the secrets sweep.

Verified: ran a real backup (45M) and confirmed all new paths are present in the archive.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-07 23:48:04 -05:00
Claude 652e44f7b3 Add JARVIS health self-check (Phase 2 reliability monitoring)
deploy/jarvis-health.sh (cron */5): detects silent failures the service watchdog cannot — stalled crons (cron.log >10min stale = 2+ missed runs), Arc jobs stuck running >30min, disk >85%, and watchdog service restarts. Writes auto-resolving rows to the alerts table (shown in admin panel) and emails myronblair@gmail.com on any NEW finding via the reactor Gmail SMTP creds.

Verified live end-to-end: injected a fake stuck job -> alert raised + email sent; cleared it -> alert auto-resolved. Installed in root crontab.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-07 20:43:37 -05:00
9 changed files with 123 additions and 5 deletions
+1
View File
@@ -0,0 +1 @@
Property of TomTom Enterprises.
+1 -1
View File
@@ -12,7 +12,7 @@ fi
SUBNET="10.48.200.0/24" SUBNET="10.48.200.0/24"
TMPFILE=$(mktemp) TMPFILE=$(mktemp)
nmap -sn --send-ip "$SUBNET" 2>/dev/null > "$TMPFILE" nmap -sn "$SUBNET" 2>/dev/null > "$TMPFILE"
if [ ! -s "$TMPFILE" ]; then if [ ! -s "$TMPFILE" ]; then
echo "$(date): nmap produced no output" >&2 echo "$(date): nmap produced no output" >&2
+7 -1
View File
@@ -197,6 +197,11 @@ function collect_all(): array {
'orbisportal' => 'https://orbis.orbishosting.com', 'orbisportal' => 'https://orbis.orbishosting.com',
'tomtomgames' => 'https://tomtomgames.com', 'tomtomgames' => 'https://tomtomgames.com',
]; ];
// Sites intentionally gated behind HTTP Basic Auth (e.g. a password-protected
// "Coming Soon" page during a rebuild) — treat these status codes as up, not down.
$expectedCodes = [
'parkerslingshotrentals' => [401],
];
$down = []; $down = [];
foreach ($sites as $key => $url) { foreach ($sites as $key => $url) {
$parsed = parse_url($url); $parsed = parse_url($url);
@@ -216,7 +221,8 @@ function collect_all(): array {
curl_exec($ch); curl_exec($ch);
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE); $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch); curl_close($ch);
$status = ($code >= 200 && $code < 400) ? 'up' : "down-$code"; $ok = ($code >= 200 && $code < 400) || in_array($code, $expectedCodes[$key] ?? [], true);
$status = $ok ? 'up' : "down-$code";
KBEngine::storeFact('sites', $key, $status, $url, 180); KBEngine::storeFact('sites', $key, $status, $url, 180);
if ($status !== 'up') $down[] = "$key($code)"; if ($status !== 'up') $down[] = "$key($code)";
} }
+5
View File
@@ -35,6 +35,11 @@ foreach ($devices as $d) {
if (!$ip) continue; if (!$ip) continue;
$discoveredIPs[] = $ip; $discoveredIPs[] = $ip;
if ($mac) {
// Device likely moved to a new IP (DHCP) — drop the stale row so it
// doesn't linger as an orphaned duplicate under the old address.
JarvisDB::execute('DELETE FROM network_devices WHERE mac=? AND ip<>?', [$mac, $ip]);
}
JarvisDB::execute( JarvisDB::execute(
'INSERT INTO network_devices (ip, mac, hostname, status, last_seen) 'INSERT INTO network_devices (ip, mac, hostname, status, last_seen)
VALUES (?,?,?,?,NOW()) VALUES (?,?,?,?,NOW())
+2 -2
View File
@@ -213,7 +213,7 @@ if ($weatherAge > 1800) {
}; };
$weatherRaw = curlGet( $weatherRaw = curlGet(
'https://wttr.in/FortWorth,TX?format=j1', 'https://wttr.in/76088?format=j1',
['User-Agent: curl/7.88 Jarvis/1.0'], ['User-Agent: curl/7.88 Jarvis/1.0'],
15 15
); );
@@ -245,7 +245,7 @@ if ($weatherAge > 1800) {
cacheStore('weather', [ cacheStore('weather', [
'source' => 'wttr.in', 'source' => 'wttr.in',
'location' => 'Fort Worth, TX', 'location' => 'Weatherford, TX',
'current' => [ 'current' => [
'temp' => (int)($cu['temp_F'] ?? 0), 'temp' => (int)($cu['temp_F'] ?? 0),
'feels' => (int)($cu['FeelsLikeF'] ?? 0), 'feels' => (int)($cu['FeelsLikeF'] ?? 0),
+8
View File
@@ -29,6 +29,14 @@ if mysqldump -u"$DB_USER" -p"$DB_PASS" "$DB_NAME" > "$TMPDIR/jarvis_db.sql" 2>>"
cp -a /etc/nginx/sites-enabled/jarvis "$TMPDIR/files/etc/nginx-site-jarvis" 2>>"$LOG" cp -a /etc/nginx/sites-enabled/jarvis "$TMPDIR/files/etc/nginx-site-jarvis" 2>>"$LOG"
cp -a /etc/systemd/system/jarvis-arc.service "$TMPDIR/files/etc/jarvis-arc.service" 2>>"$LOG" cp -a /etc/systemd/system/jarvis-arc.service "$TMPDIR/files/etc/jarvis-arc.service" 2>>"$LOG"
crontab -l > "$TMPDIR/files/etc/root-crontab.txt" 2>>"$LOG" crontab -l > "$TMPDIR/files/etc/root-crontab.txt" 2>>"$LOG"
# Phase 1/2 additions: secrets + systemd drop-ins + operational scripts
# (these live outside /var/www/jarvis and /opt/jarvis-arc, so must be
# captured explicitly or a restore comes back with no keys/DB pass).
cp -a /etc/jarvis-arc "$TMPDIR/files/etc/jarvis-arc-etc" 2>>"$LOG" # reactor.env
cp -a /etc/jarvis "$TMPDIR/files/etc/jarvis-etc" 2>>"$LOG" # db.env
cp -a /etc/systemd/system/jarvis-arc.service.d "$TMPDIR/files/etc/jarvis-arc.service.d" 2>>"$LOG"
mkdir -p "$TMPDIR/files/usr-local-bin"
cp -a /usr/local/bin/jarvis-*.sh "$TMPDIR/files/usr-local-bin/" 2>>"$LOG" # health/deploy/watchdog/netscan
tar -czf "$OUTFILE" -C "$TMPDIR" jarvis_db.sql files tar -czf "$OUTFILE" -C "$TMPDIR" jarvis_db.sql files
SIZE=$(du -sh "$OUTFILE" | cut -f1) SIZE=$(du -sh "$OUTFILE" | cut -f1)
+97
View File
@@ -0,0 +1,97 @@
#!/bin/bash
# JARVIS Health Self-Check — runs every 5 min via root cron (separate from the
# service watchdog). Detects silent failures the watchdog can't: stalled crons,
# stuck Arc jobs, low disk, and services the watchdog had to restart. Writes
# findings to the `alerts` table (auto-resolving when clear) and emails on any
# NEW finding. Added 2026-07-07 (Phase 2 reliability).
set -u
[ -r /etc/jarvis/db.env ] && . /etc/jarvis/db.env
DB_USER="jarvis_user"; DB_NAME="jarvis_db"
MYSQL=(mysql -u "$DB_USER" -p"${JARVIS_DB_PASS:-}" "$DB_NAME" -N -B -e)
CRONLOG=/var/log/jarvis/cron.log
WDLOG=/var/log/jarvis/watchdog.log
ALERT_TO="myronblair@gmail.com"
REACTOR_ENV=/etc/jarvis-arc/reactor.env
NEW_FINDINGS=""
sql() { "${MYSQL[@]}" "$1" 2>/dev/null; }
esc() { printf '%s' "$1" | sed "s/'/''/g"; }
# Raise (or keep) an alert for a condition. Emails only when it is newly raised.
# $1=source_key $2=severity $3=title $4=message
raise() {
local key sev title msg exists
key=$(esc "$1"); sev=$(esc "$2"); title=$(esc "$3"); msg=$(esc "$4")
exists=$(sql "SELECT COUNT(*) FROM alerts WHERE source_key='$key' AND resolved=0")
if [ "${exists:-0}" = "0" ]; then
sql "INSERT INTO alerts (alert_type,title,message,severity,source_key,auto_resolve,created_at)
VALUES ('health','$title','$msg','$sev','$key',1,NOW())"
NEW_FINDINGS="${NEW_FINDINGS}- [$2] $3: $4"$'\n'
fi
}
# Clear a condition's alert when it's no longer true.
clear_cond() {
local key; key=$(esc "$1")
sql "UPDATE alerts SET resolved=1, resolved_at=NOW()
WHERE source_key='$key' AND resolved=0 AND auto_resolve=1"
}
# 1) Disk usage on / > 85%
DISK=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
if [ "${DISK:-0}" -gt 85 ]; then
raise "health:disk" "critical" "Disk usage high" "Root filesystem at ${DISK}% (threshold 85%)."
else clear_cond "health:disk"; fi
# 2) Arc jobs stuck in 'running' > 30 min
STUCK=$(sql "SELECT COUNT(*) FROM arc_jobs WHERE status='running'
AND COALESCE(started_at, created_at) < NOW() - INTERVAL 30 MINUTE")
if [ "${STUCK:-0}" -gt 0 ]; then
raise "health:arc_stuck" "critical" "Arc jobs stuck" "$STUCK Arc job(s) have been 'running' for over 30 minutes."
else clear_cond "health:arc_stuck"; fi
# 3) Cron stalled — cron.log is written by facts_collector every 3 min. If it
# hasn't changed in 10 min (>2 consecutive missed runs), cron work has stopped.
if [ -f "$CRONLOG" ]; then
AGE=$(( $(date +%s) - $(stat -c %Y "$CRONLOG") ))
if [ "$AGE" -gt 600 ]; then
raise "health:cron" "critical" "Cron jobs stalled" "No cron activity in $((AGE/60)) min (facts_collector runs every 3 min) — crons appear stopped."
else clear_cond "health:cron"; fi
fi
# 4) Watched service restarted by the watchdog in the last 6 min
if [ -f "$WDLOG" ]; then
RECENT=$(awk -v cutoff="$(date -d '6 minutes ago' '+%Y-%m-%d %H:%M:%S')" \
'match($0,/^\[([0-9-]+ [0-9:]+)\]/,m){ if(m[1]>=cutoff && /restarted successfully/) print }' "$WDLOG")
if [ -n "$RECENT" ]; then
SVC=$(printf '%s' "$RECENT" | grep -oE '(nginx|php8.3-fpm|mariadb|redis-server)' | sort -u | tr '\n' ' ')
raise "health:wd_restart" "warning" "Service auto-restarted" "Watchdog restarted: ${SVC:-a service}. Investigate why it died."
else
clear_cond "health:wd_restart"
fi
fi
# Email any NEW findings via the reactor's Gmail SMTP creds.
if [ -n "$NEW_FINDINGS" ]; then
GPASS=""
[ -r "$REACTOR_ENV" ] && GPASS=$(grep -E '^GMAIL_PASS=' "$REACTOR_ENV" | cut -d= -f2-)
if [ -n "$GPASS" ]; then
GMAIL_PASS="$GPASS" ALERT_TO="$ALERT_TO" FINDINGS="$NEW_FINDINGS" python3 - <<'PY'
import os, smtplib, ssl, socket
from email.mime.text import MIMEText
user = "myronblair@gmail.com"
msg = MIMEText("JARVIS health self-check raised new alerts on %s:\n\n%s" % (socket.gethostname(), os.environ["FINDINGS"]))
msg["Subject"] = "JARVIS health alert"
msg["From"] = user; msg["To"] = os.environ["ALERT_TO"]
try:
with smtplib.SMTP("smtp.gmail.com", 587, timeout=20) as s:
s.starttls(context=ssl.create_default_context())
s.login(user, os.environ["GMAIL_PASS"])
s.send_message(msg)
print("health-email: sent")
except Exception as e:
print("health-email: FAILED", e)
PY
else
echo "health-email: no GMAIL_PASS available, skipped"
fi
fi
+1
View File
@@ -982,6 +982,7 @@ async function loadWeather() {
const d = await api('weather'); const d = await api('weather');
if (!d || !d.current) return; if (!d || !d.current) return;
const c = d.current; const c = d.current;
if (d.location) document.getElementById('weather-loc').textContent = d.location.toUpperCase();
document.getElementById('weather-temp').textContent = c.temp; document.getElementById('weather-temp').textContent = c.temp;
document.getElementById('weather-desc').textContent = (c.desc || '').toUpperCase(); document.getElementById('weather-desc').textContent = (c.desc || '').toUpperCase();
document.getElementById('weather-feels').textContent = c.feels + '°F'; document.getElementById('weather-feels').textContent = c.feels + '°F';
+1 -1
View File
@@ -84,7 +84,7 @@
<div id="leftPanel"> <div id="leftPanel">
<!-- Weather Widget --> <!-- Weather Widget -->
<div class="panel" style="flex:0 0 auto"> <div class="panel" style="flex:0 0 auto">
<div class="panel-title">WEATHER <span id="weather-loc" style="font-size:0.55rem;color:var(--text-dim)">FORT WORTH, TX</span></div> <div class="panel-title">WEATHER <span id="weather-loc" style="font-size:0.55rem;color:var(--text-dim)">WEATHERFORD, TX</span></div>
<div style="display:flex;align-items:flex-start;gap:12px;margin-bottom:8px"> <div style="display:flex;align-items:flex-start;gap:12px;margin-bottom:8px">
<div style="flex:1"> <div style="flex:1">
<div style="display:flex;align-items:baseline;gap:8px"> <div style="display:flex;align-items:baseline;gap:8px">