mirror of
https://github.com/myronblair/jarvis
synced 2026-07-27 16:22:55 -05:00
Adopt live production state as git baseline
This commit is contained in:
root
@@ -0,0 +1,2 @@
|
||||
*/3 * * * * /usr/local/lsws/lsphp85/bin/lsphp /home/jarvis.orbishosting.com/api/endpoints/facts_collector.php >> /home/jarvis.orbishosting.com/logs/cron.log 2>&1
|
||||
*/5 * * * * /usr/local/lsws/lsphp85/bin/lsphp /home/jarvis.orbishosting.com/api/endpoints/stats_cache.php >> /home/jarvis.orbishosting.com/logs/cron.log 2>&1
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=JARVIS Agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/python3 /usr/local/bin/jarvis-agent.py
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
User=root
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=JARVIS Arc Reactor
|
||||
After=network-online.target mysql.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/jarvis-arc
|
||||
ExecStart=/opt/jarvis-arc/venv/bin/python3 /opt/jarvis-arc/reactor.py
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
User=root
|
||||
StandardOutput=append:/home/jarvis.orbishosting.com/logs/arc_reactor.log
|
||||
StandardError=append:/home/jarvis.orbishosting.com/logs/arc_reactor.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# JARVIS backup — DB dump as tar.gz, admin-panel compatible
|
||||
BACKUP_DIR="/var/backups/jarvis"
|
||||
LOG="$BACKUP_DIR/backup.log"
|
||||
LOCK="$BACKUP_DIR/backup.lock"
|
||||
DB_NAME="jarvis_db"
|
||||
DB_USER="jarvis_user"
|
||||
DB_PASS="J4rv1s_Pr0t0c0l_2026!"
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
OUTFILE="$BACKUP_DIR/jarvis_backup_${TIMESTAMP}.tar.gz"
|
||||
TMPDIR=$(mktemp -d)
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
touch "$LOCK"
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup..." >> "$LOG"
|
||||
|
||||
cleanup() { rm -rf "$TMPDIR"; rm -f "$LOCK"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
if mysqldump -u"$DB_USER" -p"$DB_PASS" "$DB_NAME" > "$TMPDIR/jarvis_db.sql" 2>>"$LOG"; then
|
||||
tar -czf "$OUTFILE" -C "$TMPDIR" jarvis_db.sql
|
||||
SIZE=$(du -sh "$OUTFILE" | cut -f1)
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup OK: $(basename "$OUTFILE") ($SIYE)" >> "$LOG"
|
||||
else
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: mysqldump failed" >> "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
find "$BACKUP_DIR" -name "jarvis_backup_*.tar.gz" -mtime +7 -delete
|
||||
COUNT=$(ls "$BACKUP_DIR"/jarvis_backup_*.tar.gz 2>/dev/null | wc -l)
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] Done. Files retained: $COUNT" >> "$LOG"
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/bin/bash
|
||||
# JARVIS Auto-Deploy Runner — processes GitHub webhook queue every minute.
|
||||
# Validates PHP syntax before deploying; auto-reverts on bad code.
|
||||
# Restarts OLS after JARVIS deploys to pick up PHP changes.
|
||||
|
||||
QUEUE=/tmp/jarvis-deploy-queue.txt
|
||||
LOG=/home/jarvis.orbishosting.com/logs/deploy.log
|
||||
PHP=/usr/bin/php8.3
|
||||
|
||||
TS() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
log() { echo "[$(TS)] $1" >> "$LOG"; }
|
||||
|
||||
[ ! -f "$QUEUE" ] && exit 0
|
||||
[ ! -s "$QUEUE" ] && exit 0
|
||||
|
||||
# Atomically take ownership of the queue via rename — prevents TOCTOU loss of
|
||||
# entries written between a cat and truncate
|
||||
PROCESSING="${QUEUE}.processing"
|
||||
mv "$QUEUE" "$PROCESSING" 2>/dev/null || exit 0
|
||||
SNAPSHOT=$(cat "$PROCESSING")
|
||||
rm -f "$PROCESSING"
|
||||
|
||||
while IFS= read -r path; do
|
||||
[ -z "$path" ] && continue
|
||||
[ ! -d "$path/.git" ] && log "SKIP $path — not a git repo" && continue
|
||||
|
||||
log "Deploying $path"
|
||||
cd "$path" || continue
|
||||
|
||||
BEFORE=$(git rev-parse HEAD 2>/dev/null)
|
||||
git fetch origin main >> "$LOG" 2>&1
|
||||
REMOTE=$(git rev-parse origin/main 2>/dev/null)
|
||||
|
||||
if [ "$BEFORE" = "$REMOTE" ]; then
|
||||
log "Already up to date: $path"
|
||||
continue
|
||||
fi
|
||||
|
||||
git pull origin main >> "$LOG" 2>&1
|
||||
AFTER=$(git rev-parse HEAD 2>/dev/null)
|
||||
CHANGED=$(git diff --name-only "$BEFORE" "$AFTER" 2>/dev/null)
|
||||
|
||||
# PHP syntax validation — check every changed .php file
|
||||
SYNTAX_OK=true
|
||||
BAD_FILE=""
|
||||
while IFS= read -r f; do
|
||||
[[ "$f" != *.php ]] && continue
|
||||
[ ! -f "$f" ] && continue
|
||||
if ! $PHP -l "$f" > /dev/null 2>&1; then
|
||||
SYNTAX_OK=false
|
||||
BAD_FILE="$f"
|
||||
break
|
||||
fi
|
||||
done <<< "$CHANGED"
|
||||
|
||||
if [ "$SYNTAX_OK" = false ]; then
|
||||
log "SYNTAX ERROR in $BAD_FILE — reverting locally and pushing revert to GitHub"
|
||||
git reset --hard "$BEFORE" >> "$LOG" 2>&1
|
||||
# Push the revert so GitHub matches the live server — prevents infinite re-deploy loop
|
||||
git push --force origin HEAD:main >> "$LOG" 2>&1
|
||||
PUSH_EXIT=$?
|
||||
if [ $PUSH_EXIT -ne 0 ]; then
|
||||
log "WARNING: Force-push of revert failed (exit $PUSH_EXIT) — bad commit still on GitHub"
|
||||
fi
|
||||
# Insert alert into JARVIS DB
|
||||
BAD_ESCAPED=$(printf '%s' "$BAD_FILE" | sed "s/'/\\\\\\'/g")
|
||||
mysql -u jarvis_user -pJ4rv1s_Pr0t0c0l_2026! jarvis_db -se \
|
||||
"INSERT INTO alerts (alert_type,title,message,severity)
|
||||
VALUES ('deploy_fail','Deploy reverted: syntax error',
|
||||
'PHP syntax error in $BAD_ESCAPED. Commit $AFTER was reverted and force-pushed to GitHub.','critical');" 2>/dev/null
|
||||
log "Reverted. Bad commit: $AFTER"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Deploy OK ($BEFORE -> $AFTER): $path"
|
||||
log "Changed: $(echo "$CHANGED" | tr '\n' ' ')"
|
||||
|
||||
# Restart OLS after any JARVIS deploy to pick up PHP changes
|
||||
if [[ "$path" == *"jarvis"* ]]; then
|
||||
systemctl reload lsws 2>/dev/null || systemctl restart lsws 2>/dev/null
|
||||
log "OLS reloaded for JARVIS deploy"
|
||||
|
||||
# Patch live config.php with correct Ollama host/model and Groq search model
|
||||
CONFIG="$path/api/config.php"
|
||||
if [ -f "$CONFIG" ]; then
|
||||
sed -i "s|http://10\.48\.200\.95:11434|http://10.48.200.210:11434|g" "$CONFIG"
|
||||
sed -i "s|'llama3\.2:1b'|'llama3.1:8b'|g" "$CONFIG"
|
||||
sed -i "s|\"llama3\.2:1b\"|\"llama3.1:8b\"|g" "$CONFIG"
|
||||
sed -i "s|'llama3\.1:70b'|'llama3.1:8b'|g" "$CONFIG"
|
||||
sed -i "s|\"llama3\.1:70b\"|\"llama3.1:8b\"|g" "$CONFIG"
|
||||
sed -i "s|'groq/compound-mini'|'compound-beta-mini'|g;s|\"groq/compound-mini\"|\"compound-beta-mini\"|g" "$CONFIG"
|
||||
log "Patched config.php: Ollama IP→.210, model→llama3.1:8b, Groq search→compound-beta-mini"
|
||||
fi
|
||||
|
||||
# Self-install the deploy script on every run
|
||||
cp "$path/deploy/jarvis-deploy.sh" /usr/local/bin/jarvis-deploy.sh 2>/dev/null && chmod +x /usr/local/bin/jarvis-deploy.sh
|
||||
|
||||
# Sync reactor.py + service file to runtime location if they changed
|
||||
if echo "$CHANGED" | grep -q 'deploy/reactor.py\|deploy/jarvis-arc.service\|deploy/requirements.txt'; then
|
||||
mkdir -p /opt/jarvis-arc /home/jarvis.orbishosting.com/logs
|
||||
cp "$path/deploy/reactor.py" /opt/jarvis-arc/reactor.py
|
||||
cp "$path/deploy/requirements.txt" /opt/jarvis-arc/requirements.txt 2>/dev/null
|
||||
# Bootstrap venv if it doesn't exist
|
||||
if [ ! -f /opt/jarvis-arc/venv/bin/activate ]; then
|
||||
log "Arc Reactor venv missing — creating and installing packages"
|
||||
python3 -m venv /opt/jarvis-arc/venv
|
||||
/opt/jarvis-arc/venv/bin/pip install -q -r /opt/jarvis-arc/requirements.txt
|
||||
log "Arc Reactor venv ready"
|
||||
fi
|
||||
if echo "$CHANGED" | grep -q 'deploy/jarvis-arc.service'; then
|
||||
cp "$path/deploy/jarvis-arc.service" /etc/systemd/system/jarvis-arc.service
|
||||
systemctl daemon-reload
|
||||
systemctl enable jarvis-arc
|
||||
log "Arc Reactor service file installed and enabled"
|
||||
fi
|
||||
systemctl restart jarvis-arc 2>/dev/null || \
|
||||
bash -c "pkill -f reactor.py 2>/dev/null; sleep 1; cd /opt/jarvis-arc && source venv/bin/activate && nohup python3 reactor.py >> /home/jarvis.orbishosting.com/logs/arc_reactor.log 2>&1 &"
|
||||
log "Arc Reactor updated and restarted"
|
||||
fi
|
||||
fi
|
||||
|
||||
done <<< "$SNAPSHOT"
|
||||
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/bin/bash
|
||||
# JARVIS Self-Healing Watchdog — runs every 5 min via root cron
|
||||
# Checks: lsws, mysql, redis, JARVIS HTTP, disk, memory
|
||||
# Auto-heals: restarts failed services, restarts offline Proxmox VM agents
|
||||
# Logs to: /home/jarvis.orbishosting.com/logs/watchdog.log
|
||||
|
||||
LOG=/home/jarvis.orbishosting.com/logs/watchdog.log
|
||||
MYSQL="mysql -u jarvis_user -pJ4rv1s_Pr0t0c0l_2026! jarvis_db -se"
|
||||
TS() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
|
||||
log() { echo "[$(TS)] $1" >> "$LOG"; }
|
||||
|
||||
# Escape single quotes for MySQL string interpolation in bash
|
||||
sql_esc() { printf '%s' "$1" | sed "s/'/\\\\''/g"; }
|
||||
|
||||
alert() {
|
||||
local type="$1" title="$2" msg="$3" sev="${4:-warning}"
|
||||
local e_type e_title e_msg e_sev
|
||||
e_type=$(sql_esc "$type"); e_title=$(sql_esc "$title")
|
||||
e_msg=$(sql_esc "$msg"); e_sev=$(sql_esc "$sev")
|
||||
$MYSQL "INSERT IGNORE INTO alerts (alert_type,title,message,severity,source_key,auto_resolve)
|
||||
VALUES ('$e_type','$e_title','$e_msg','$e_sev','watchdog:$e_type',1);" 2>/dev/null
|
||||
}
|
||||
|
||||
resolve() {
|
||||
local e_key
|
||||
e_key=$(sql_esc "$1")
|
||||
$MYSQL "UPDATE alerts SET resolved=1,resolved_at=NOW()
|
||||
WHERE source_key='watchdog:$e_key' AND resolved=0;" 2>/dev/null
|
||||
}
|
||||
|
||||
# ── Service health ─────────────────────────────────────────────────────────────
|
||||
for SVC in lsws mysql redis; do
|
||||
if ! systemctl is-active --quiet "$SVC"; then
|
||||
log "HEAL: $SVC is down — restarting"
|
||||
systemctl restart "$SVC"
|
||||
if systemctl is-active --quiet "$SVC"; then
|
||||
log "HEAL: $SVC restarted successfully"
|
||||
alert "service_down" "$SVC restarted" "JARVIS watchdog restarted $SVC which was stopped." "warning"
|
||||
else
|
||||
log "ERROR: $SVC failed to restart"
|
||||
alert "service_down" "$SVC failed to restart" "$SVC is down and could not be restarted automatically." "critical"
|
||||
fi
|
||||
else
|
||||
resolve "service_down_$SVC"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── JARVIS HTTP self-check ─────────────────────────────────────────────────────
|
||||
HTTP_CODE=$(curl -sk -o /dev/null -w "%{http_code}" --max-time 10 https://jarvis.orbishosting.com/api.php 2>/dev/null)
|
||||
if [[ "$HTTP_CODE" == "5"* ]] || [[ -z "$HTTP_CODE" ]]; then
|
||||
log "HEAL: JARVIS HTTP returned $HTTP_CODE — restarting lsws"
|
||||
systemctl restart lsws
|
||||
alert "jarvis_http" "JARVIS HTTP error — restarted OLS" "JARVIS returned HTTP $HTTP_CODE. OpenLiteSpeed was restarted." "critical"
|
||||
else
|
||||
resolve "jarvis_http"
|
||||
fi
|
||||
|
||||
# ── Disk usage ─────────────────────────────────────────────────────────────────
|
||||
DISK_PCT=$(df / | awk 'NR==2{print $5}' | tr -d '%')
|
||||
if [ "$DISK_PCT" -ge 90 ]; then
|
||||
log "ALERT: Disk at ${DISK_PCT}% (critical)"
|
||||
alert "disk_critical" "Disk ${DISK_PCT}% full on DO server" "Root filesystem is ${DISK_PCT}% full. Immediate cleanup required." "critical"
|
||||
elif [ "$DISK_PCT" -ge 80 ]; then
|
||||
log "WARN: Disk at ${DISK_PCT}%"
|
||||
alert "disk_warning" "Disk ${DISK_PCT}% full on DO server" "Root filesystem is ${DISK_PCT}% full." "warning"
|
||||
else
|
||||
$MYSQL "UPDATE alerts SET resolved=1,resolved_at=NOW() WHERE source_key IN ('watchdog:disk_critical','watchdog:disk_warning') AND resolved=0;" 2>/dev/null
|
||||
fi
|
||||
|
||||
# ── Memory usage ──────────────────────────────────────────────────────────────
|
||||
MEM_TOTAL=$(grep MemTotal /proc/meminfo | awk '{print $2}')
|
||||
MEM_AVAIL=$(grep MemAvailable /proc/meminfo | awk '{print $2}')
|
||||
MEM_PCT=$(( (MEM_TOTAL - MEM_AVAIL) * 100 / MEM_TOTAL ))
|
||||
if [ "$MEM_PCT" -ge 90 ]; then
|
||||
log "ALERT: Memory at ${MEM_PCT}%"
|
||||
alert "mem_critical" "Memory ${MEM_PCT}% used on DO server" "DO server memory is ${MEM_PCT}% used." "critical"
|
||||
fi
|
||||
|
||||
# ── Offline agent auto-restart (Proxmox VMs only) ─────────────────────────────
|
||||
# Map: agent_id → [proxmox_ip, vmid]
|
||||
declare -A AGENT_PVE=(
|
||||
["ollama_vm"]="10.48.200.90 210"
|
||||
["ha_vm"]="10.48.200.90 101"
|
||||
["networkbackup_vm"]="10.48.200.91 302"
|
||||
)
|
||||
|
||||
OFFLINE=$($MYSQL "SELECT agent_id FROM registered_agents
|
||||
WHERE status='offline' AND last_seen < DATE_SUB(NOW(), INTERVAL 5 MINUTE)
|
||||
AND agent_type='linux';" 2>/dev/null)
|
||||
|
||||
for AID in $OFFLINE; do
|
||||
# Check if we have a Proxmox mapping for this agent
|
||||
for KEY in "${!AGENT_PVE[@]}"; do
|
||||
if [[ "$AID" == *"$KEY"* ]] || [[ "$KEY" == *"$AID"* ]]; then
|
||||
PVE_INFO=(${AGENT_PVE[$KEY]})
|
||||
PVE_IP="${PVE_INFO[0]}"
|
||||
VMID="${PVE_INFO[1]}"
|
||||
log "HEAL: Attempting to restart jarvis-agent on $AID (VM $VMID @ $PVE_IP)"
|
||||
sshpass -p 'Joker1974!!!' ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
||||
root@"$PVE_IP" \
|
||||
"qm guest exec $VMID -- systemctl restart jarvis-agent" 2>/dev/null
|
||||
log "HEAL: Restart command sent to $AID (exit: $?)"
|
||||
alert "agent_offline" "Auto-restarted agent: $AID" \
|
||||
"Agent $AID was offline. JARVIS watchdog sent restart command via Proxmox." "warning"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# ── Deploy log rotation (keep last 1000 lines) ────────────────────────────────
|
||||
for LOGFILE in "$LOG" /home/jarvis.orbishosting.com/logs/deploy.log /home/jarvis.orbishosting.com/logs/cron.log; do
|
||||
[ -f "$LOGFILE" ] || continue
|
||||
LINES=$(wc -l < "$LOGFILE")
|
||||
if [ "$LINES" -gt 1000 ]; then
|
||||
tail -500 "$LOGFILE" > "${LOGFILE}.tmp" && mv "${LOGFILE}.tmp" "$LOGFILE"
|
||||
fi
|
||||
done
|
||||
+2789
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
aiomysql
|
||||
aiohttp
|
||||
anthropic
|
||||
trafilatura
|
||||
Reference in New Issue
Block a user