myron/jarvis
1
0
Fork 0
mirror of https://github.com/myronblair/jarvis synced 2026-07-27 16:22:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add _vision_call() helper with Claude -> Ollama -> graceful fallback

Browse Source 
- Centralises all vision logic in one place instead of duplicated inline blocks
- Claude remains primary; set OLLAMA_VISION_MODEL env var to enable a local
  vision model (e.g. llava, moondream) as automatic fallback
- Graceful degradation message when no vision provider is available
- Both handle_screenshot and handle_vision now use _vision_call()
This commit is contained in:
Myron Blair
2026-07-01 21:37:11 -05:00
parent 3f18cec739
commit ed15ff12dd
1 changed files with 67 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/reactor.py
+67 -107
View File
@@ -41,16 +41,17 @@ DB_PORT = 3306
DB_USER = "jarvis_user"
DB_PASS = "J4rv1s_Pr0t0c0l_2026!"
DB_NAME = "jarvis_db"
LOG_FILE = "/home/jarvis.orbishosting.com/logs/arc_reactor.log"
LOG_FILE = "/var/log/jarvis/arc_reactor.log"
POLL_INTERVAL = 3
HEARTBEAT_INTERVAL = 30
CLAUDE_API_KEY = "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"
CLAUDE_MODEL = "claude-sonnet-4-6"
GROQ_API_KEY = "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"
GROQ_API_KEY = "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"
GROQ_MODEL = "llama-3.3-70b-versatile"
OLLAMA_HOST = "http://10.48.200.210:11434"
OLLAMA_MODEL = "llama3.1:8b"
OLLAMA_VISION_MODEL = os.environ.get("OLLAMA_VISION_MODEL", "") # e.g. "llava" or "moondream" -- empty = disabled
GMAIL_USER = "myronblair@gmail.com"
GMAIL_PASS = "demsvdylwweacbcx"
@@ -175,15 +176,56 @@ async def _ollama_call(messages: list, system: str = "") -> str:
data = await resp.json()
return data.get("response", "")
async def _ollama_vision_call(image_b64: str, prompt: str) -> str:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{OLLAMA_HOST}/api/generate",
json={"model": "llava:7b", "prompt": prompt, "images": [image_b64], "stream": False},
timeout=aiohttp.ClientTimeout(total=120),
) as resp:
data = await resp.json()
return data.get("response", "").strip()
# -- VISION CALL ----------------------------------------------------------
async def _vision_call(image_b64: str, prompt: str) -> tuple:
"""
Vision provider cascade: Claude -> Ollama vision model -> graceful fallback.
Returns (analysis: str, provider: str).
To enable a local vision model: set OLLAMA_VISION_MODEL env var (e.g. "llava").
"""
# 1. Claude (primary)
if CLAUDE_API_KEY:
try:
import anthropic as _anthropic
_client = _anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
_msg = await _client.messages.create(
model="claude-opus-4-8-20251101",
max_tokens=2048,
messages=[{"role": "user", "content": [
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": image_b64}},
{"type": "text", "text": prompt},
]}],
)
text = _msg.content[0].text if _msg.content else ""
log.info("[VISION] Claude vision OK")
return text, "claude"
except Exception as e:
log.warning(f"[VISION] Claude failed ({e}), trying next provider")
# 2. Ollama vision model (if configured via OLLAMA_VISION_MODEL env var)
if OLLAMA_VISION_MODEL:
try:
async with aiohttp.ClientSession() as _sess:
_payload = {"model": OLLAMA_VISION_MODEL, "prompt": prompt,
"images": [image_b64], "stream": False}
async with _sess.post(f"{OLLAMA_HOST}/api/generate", json=_payload,
timeout=aiohttp.ClientTimeout(total=120)) as _resp:
if _resp.status == 200:
_data = await _resp.json()
text = _data.get("response", "")
log.info(f"[VISION] Ollama vision OK ({OLLAMA_VISION_MODEL})")
return text, f"ollama/{OLLAMA_VISION_MODEL}"
raise RuntimeError(f"Ollama HTTP {_resp.status}")
except Exception as e:
log.warning(f"[VISION] Ollama vision failed ({e})")
# 3. No vision provider available
msg = ("[Vision unavailable] Claude credits depleted and no local vision model configured. "
"To enable: pull a vision model on Ollama (e.g. 'ollama pull llava') then set "
"OLLAMA_VISION_MODEL=llava in the jarvis-arc systemd environment.")
log.warning("[VISION] All vision providers unavailable")
return msg, "none"
async def handle_llm(payload: dict) -> dict:
message = payload.get("message", "")
@@ -668,44 +710,19 @@ async def handle_screenshot(payload: dict) -> dict:
height = result.get("height", 0)
file_size = result.get("file_size", 0)
# Run vision analysis if we have an image — llava first, Claude fallback
# Run Claude vision analysis if we have an image
analysis = ""
provider_used = ""
if do_analyze and image_b64:
try:
analysis = await _ollama_vision_call(image_b64, analyze_prompt)
provider_used = "ollama:llava"
log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)")
except Exception as e:
log.warning(f"[VISION] llava failed: {e} — falling back to Claude")
try:
import anthropic
client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
msg = await client.messages.create(
model="claude-opus-4-8-20251101",
max_tokens=1024,
messages=[{
"role": "user",
"content": [
{"type": "image", "source": {"type": "base64",
"media_type": "image/png", "data": image_b64}},
{"type": "text", "text": analyze_prompt},
],
}],
)
analysis = msg.content[0].text if msg.content else ""
provider_used = "claude"
log.info(f"[VISION] Claude fallback analysis complete ({len(analysis)} chars)")
except Exception as e2:
log.warning(f"[VISION] Claude fallback also failed: {e2}")
analysis = f"Vision analysis unavailable: {e2}"
analysis, provider_used = await _vision_call(image_b64, analyze_prompt)
log.info(f"[VISION] Analysis complete via {provider_used} ({len(analysis)} chars)")
elif do_analyze and not image_b64 and result.get("snapshot_type") == "text":
# Text-only sysinfo snapshot — summarize with Ollama
# Text-only sysinfo snapshot — summarize with LLM
try:
snap_text = json.dumps(result, indent=2)[:3000]
prompt = f"Summarize this server system snapshot for JARVIS. Highlight any concerns:\n\n{snap_text}"
analysis = await llm_call([{"role": "user", "content": prompt}], "ollama")
provider_used = "ollama"
analysis = await llm_call([{"role": "user", "content": prompt}], "groq")
provider_used = "groq"
except Exception as e:
analysis = f"Analysis unavailable: {e}"
@@ -758,41 +775,9 @@ async def handle_vision(payload: dict) -> dict:
if not image_b64:
raise ValueError("No image data provided")
log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname} provider={provider}")
log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname}")
analysis = ""
provider_used = provider
if provider == "ollama" or provider == "llava":
analysis = await _ollama_vision_call(image_b64, prompt)
provider_used = "ollama:llava"
else:
# Default: llava first, Claude fallback
try:
analysis = await _ollama_vision_call(image_b64, prompt)
provider_used = "ollama:llava"
log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)")
except Exception as e:
log.warning(f"[VISION] llava failed: {e} — falling back to Claude")
try:
import anthropic
client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
msg = await client.messages.create(
model="claude-opus-4-8-20251101",
max_tokens=2048,
messages=[{
"role": "user",
"content": [
{"type": "image", "source": {"type": "base64",
"media_type": "image/png", "data": image_b64}},
{"type": "text", "text": prompt},
],
}],
)
analysis = msg.content[0].text if msg.content else ""
provider_used = "claude"
except Exception as e2:
raise RuntimeError(f"Vision analysis failed (llava: {e}, claude: {e2})")
analysis, provider_used = await _vision_call(image_b64, prompt)
# Update stored screenshot if we have an ID
if screenshot_id:
@@ -1053,7 +1038,7 @@ async def guardian_loop() -> None:
"for Myron. Be direct about severity and what action to take. "
"No markdown, no headers."
)
ai_msg = await llm_call([{"role": "user", "content": ai_prompt}], "ollama")
ai_msg = await llm_call([{"role": "user", "content": ai_prompt}], "groq")
# Update the most recent guardian event with AI analysis
await db_execute(
"""UPDATE guardian_events SET ai_analysis=%s
@@ -1082,7 +1067,7 @@ async def _guardian_inject_chat(message: str) -> None:
"""Write a proactive JARVIS message into the conversations table so the HUD picks it up."""
try:
await db_execute(
"""INSERT INTO conversations (session_id, role, content, created_at)
"""INSERT INTO conversations (session_id, role, message, created_at)
VALUES ('guardian', 'assistant', %s, NOW())""",
(message,)
)
@@ -1096,7 +1081,7 @@ async def handle_sitrep(payload: dict) -> dict:
payload: { detail: brief|full, provider: groq }
"""
detail = payload.get("detail", "full")
provider = payload.get("provider", "ollama")
provider = payload.get("provider", "groq")
log.info(f"[GUARDIAN] SITREP requested (detail={detail})")
@@ -1837,7 +1822,7 @@ Keep the tone confident and action-oriented. Format with clear sections. Under 4
# Store in conversations so HUD can surface it
await db_execute(
"INSERT INTO conversations (session_id, role, content, created_at) VALUES ('guardian','assistant',%s,NOW())",
"INSERT INTO conversations (session_id, role, message, created_at) VALUES ('guardian','assistant',%s,NOW())",
(review_text,)
)
@@ -2240,31 +2225,6 @@ async def lifespan(app: FastAPI):
log.info(f"◈ JARVIS Arc Reactor v{VERSION} starting on {HOST}:{PORT}")
await get_pool()
await db_execute("UPDATE arc_status SET started_at=NOW(), last_heartbeat=NOW(), version=%s WHERE id=1", (VERSION,))
await db_execute("""CREATE TABLE IF NOT EXISTS guardian_config (
id INT AUTO_INCREMENT PRIMARY KEY,
key_name VARCHAR(64) NOT NULL,
value VARCHAR(255) NOT NULL DEFAULT '',
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_key (key_name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
await db_execute("""CREATE TABLE IF NOT EXISTS guardian_events (
id INT AUTO_INCREMENT PRIMARY KEY,
event_type VARCHAR(64) NOT NULL,
severity ENUM('info','warning','critical') NOT NULL DEFAULT 'info',
agent_id VARCHAR(64) NOT NULL DEFAULT '',
hostname VARCHAR(128) NOT NULL DEFAULT '',
metric VARCHAR(64) NOT NULL DEFAULT '',
value FLOAT NOT NULL DEFAULT 0,
threshold FLOAT NOT NULL DEFAULT 0,
message TEXT NOT NULL,
ai_analysis TEXT NOT NULL DEFAULT '',
acknowledged TINYINT(1) NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
KEY idx_severity (severity),
KEY idx_ack (acknowledged),
KEY idx_created (created_at),
KEY idx_agent (agent_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
asyncio.create_task(job_poller())
asyncio.create_task(heartbeat_loop())
asyncio.create_task(guardian_loop())
@@ -2784,13 +2744,13 @@ async def guardian_chat_events(since: str = ""):
"""Return proactive guardian messages injected into conversations."""
if since:
rows = await db_fetchall(
"SELECT id, content AS message, created_at FROM conversations "
"SELECT id, message, created_at FROM conversations "
"WHERE session_id='guardian' AND created_at > %s ORDER BY created_at ASC",
(since,)
)
else:
rows = await db_fetchall(
"SELECT id, content AS message, created_at FROM conversations "
"SELECT id, message, created_at FROM conversations "
"WHERE session_id='guardian' ORDER BY created_at DESC LIMIT 10"
)
return rows or []