From ed15ff12dd162c37d03d7c515b4fc883e98d0473 Mon Sep 17 00:00:00 2001 From: Myron Blair Date: Wed, 1 Jul 2026 21:37:11 -0500 Subject: [PATCH] Add _vision_call() helper with Claude -> Ollama -> graceful fallback - Centralises all vision logic in one place instead of duplicated inline blocks - Claude remains primary; set OLLAMA_VISION_MODEL env var to enable a local vision model (e.g. llava, moondream) as automatic fallback - Graceful degradation message when no vision provider is available - Both handle_screenshot and handle_vision now use _vision_call() --- deploy/reactor.py | 174 ++++++++++++++++++---------------------------- 1 file changed, 67 insertions(+), 107 deletions(-) diff --git a/deploy/reactor.py b/deploy/reactor.py index 7f53cef..6939e19 100644 --- a/deploy/reactor.py +++ b/deploy/reactor.py @@ -41,16 +41,17 @@ DB_PORT = 3306 DB_USER = "jarvis_user" DB_PASS = "J4rv1s_Pr0t0c0l_2026!" DB_NAME = "jarvis_db" -LOG_FILE = "/home/jarvis.orbishosting.com/logs/arc_reactor.log" +LOG_FILE = "/var/log/jarvis/arc_reactor.log" POLL_INTERVAL = 3 HEARTBEAT_INTERVAL = 30 CLAUDE_API_KEY = "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" CLAUDE_MODEL = "claude-sonnet-4-6" -GROQ_API_KEY = "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" +GROQ_API_KEY = "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" GROQ_MODEL = "llama-3.3-70b-versatile" OLLAMA_HOST = "http://10.48.200.210:11434" OLLAMA_MODEL = "llama3.1:8b" +OLLAMA_VISION_MODEL = os.environ.get("OLLAMA_VISION_MODEL", "") # e.g. "llava" or "moondream" -- empty = disabled GMAIL_USER = "myronblair@gmail.com" GMAIL_PASS = "demsvdylwweacbcx" @@ -175,15 +176,56 @@ async def _ollama_call(messages: list, system: str = "") -> str: data = await resp.json() return data.get("response", "") -async def _ollama_vision_call(image_b64: str, prompt: str) -> str: - async with aiohttp.ClientSession() as session: - async with session.post( - f"{OLLAMA_HOST}/api/generate", - json={"model": "llava:7b", "prompt": prompt, "images": [image_b64], "stream": False}, - timeout=aiohttp.ClientTimeout(total=120), - ) as resp: - data = await resp.json() - return data.get("response", "").strip() + +# -- VISION CALL ---------------------------------------------------------- +async def _vision_call(image_b64: str, prompt: str) -> tuple: + """ + Vision provider cascade: Claude -> Ollama vision model -> graceful fallback. + Returns (analysis: str, provider: str). + To enable a local vision model: set OLLAMA_VISION_MODEL env var (e.g. "llava"). + """ + # 1. Claude (primary) + if CLAUDE_API_KEY: + try: + import anthropic as _anthropic + _client = _anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) + _msg = await _client.messages.create( + model="claude-opus-4-8-20251101", + max_tokens=2048, + messages=[{"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": image_b64}}, + {"type": "text", "text": prompt}, + ]}], + ) + text = _msg.content[0].text if _msg.content else "" + log.info("[VISION] Claude vision OK") + return text, "claude" + except Exception as e: + log.warning(f"[VISION] Claude failed ({e}), trying next provider") + + # 2. Ollama vision model (if configured via OLLAMA_VISION_MODEL env var) + if OLLAMA_VISION_MODEL: + try: + async with aiohttp.ClientSession() as _sess: + _payload = {"model": OLLAMA_VISION_MODEL, "prompt": prompt, + "images": [image_b64], "stream": False} + async with _sess.post(f"{OLLAMA_HOST}/api/generate", json=_payload, + timeout=aiohttp.ClientTimeout(total=120)) as _resp: + if _resp.status == 200: + _data = await _resp.json() + text = _data.get("response", "") + log.info(f"[VISION] Ollama vision OK ({OLLAMA_VISION_MODEL})") + return text, f"ollama/{OLLAMA_VISION_MODEL}" + raise RuntimeError(f"Ollama HTTP {_resp.status}") + except Exception as e: + log.warning(f"[VISION] Ollama vision failed ({e})") + + # 3. No vision provider available + msg = ("[Vision unavailable] Claude credits depleted and no local vision model configured. " + "To enable: pull a vision model on Ollama (e.g. 'ollama pull llava') then set " + "OLLAMA_VISION_MODEL=llava in the jarvis-arc systemd environment.") + log.warning("[VISION] All vision providers unavailable") + return msg, "none" async def handle_llm(payload: dict) -> dict: message = payload.get("message", "") @@ -668,44 +710,19 @@ async def handle_screenshot(payload: dict) -> dict: height = result.get("height", 0) file_size = result.get("file_size", 0) - # Run vision analysis if we have an image — llava first, Claude fallback + # Run Claude vision analysis if we have an image analysis = "" provider_used = "" if do_analyze and image_b64: - try: - analysis = await _ollama_vision_call(image_b64, analyze_prompt) - provider_used = "ollama:llava" - log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)") - except Exception as e: - log.warning(f"[VISION] llava failed: {e} — falling back to Claude") - try: - import anthropic - client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) - msg = await client.messages.create( - model="claude-opus-4-8-20251101", - max_tokens=1024, - messages=[{ - "role": "user", - "content": [ - {"type": "image", "source": {"type": "base64", - "media_type": "image/png", "data": image_b64}}, - {"type": "text", "text": analyze_prompt}, - ], - }], - ) - analysis = msg.content[0].text if msg.content else "" - provider_used = "claude" - log.info(f"[VISION] Claude fallback analysis complete ({len(analysis)} chars)") - except Exception as e2: - log.warning(f"[VISION] Claude fallback also failed: {e2}") - analysis = f"Vision analysis unavailable: {e2}" + analysis, provider_used = await _vision_call(image_b64, analyze_prompt) + log.info(f"[VISION] Analysis complete via {provider_used} ({len(analysis)} chars)") elif do_analyze and not image_b64 and result.get("snapshot_type") == "text": - # Text-only sysinfo snapshot — summarize with Ollama + # Text-only sysinfo snapshot — summarize with LLM try: snap_text = json.dumps(result, indent=2)[:3000] prompt = f"Summarize this server system snapshot for JARVIS. Highlight any concerns:\n\n{snap_text}" - analysis = await llm_call([{"role": "user", "content": prompt}], "ollama") - provider_used = "ollama" + analysis = await llm_call([{"role": "user", "content": prompt}], "groq") + provider_used = "groq" except Exception as e: analysis = f"Analysis unavailable: {e}" @@ -758,41 +775,9 @@ async def handle_vision(payload: dict) -> dict: if not image_b64: raise ValueError("No image data provided") - log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname} provider={provider}") + log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname}") - analysis = "" - provider_used = provider - - if provider == "ollama" or provider == "llava": - analysis = await _ollama_vision_call(image_b64, prompt) - provider_used = "ollama:llava" - else: - # Default: llava first, Claude fallback - try: - analysis = await _ollama_vision_call(image_b64, prompt) - provider_used = "ollama:llava" - log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)") - except Exception as e: - log.warning(f"[VISION] llava failed: {e} — falling back to Claude") - try: - import anthropic - client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) - msg = await client.messages.create( - model="claude-opus-4-8-20251101", - max_tokens=2048, - messages=[{ - "role": "user", - "content": [ - {"type": "image", "source": {"type": "base64", - "media_type": "image/png", "data": image_b64}}, - {"type": "text", "text": prompt}, - ], - }], - ) - analysis = msg.content[0].text if msg.content else "" - provider_used = "claude" - except Exception as e2: - raise RuntimeError(f"Vision analysis failed (llava: {e}, claude: {e2})") + analysis, provider_used = await _vision_call(image_b64, prompt) # Update stored screenshot if we have an ID if screenshot_id: @@ -1053,7 +1038,7 @@ async def guardian_loop() -> None: "for Myron. Be direct about severity and what action to take. " "No markdown, no headers." ) - ai_msg = await llm_call([{"role": "user", "content": ai_prompt}], "ollama") + ai_msg = await llm_call([{"role": "user", "content": ai_prompt}], "groq") # Update the most recent guardian event with AI analysis await db_execute( """UPDATE guardian_events SET ai_analysis=%s @@ -1082,7 +1067,7 @@ async def _guardian_inject_chat(message: str) -> None: """Write a proactive JARVIS message into the conversations table so the HUD picks it up.""" try: await db_execute( - """INSERT INTO conversations (session_id, role, content, created_at) + """INSERT INTO conversations (session_id, role, message, created_at) VALUES ('guardian', 'assistant', %s, NOW())""", (message,) ) @@ -1096,7 +1081,7 @@ async def handle_sitrep(payload: dict) -> dict: payload: { detail: brief|full, provider: groq } """ detail = payload.get("detail", "full") - provider = payload.get("provider", "ollama") + provider = payload.get("provider", "groq") log.info(f"[GUARDIAN] SITREP requested (detail={detail})") @@ -1837,7 +1822,7 @@ Keep the tone confident and action-oriented. Format with clear sections. Under 4 # Store in conversations so HUD can surface it await db_execute( - "INSERT INTO conversations (session_id, role, content, created_at) VALUES ('guardian','assistant',%s,NOW())", + "INSERT INTO conversations (session_id, role, message, created_at) VALUES ('guardian','assistant',%s,NOW())", (review_text,) ) @@ -2240,31 +2225,6 @@ async def lifespan(app: FastAPI): log.info(f"◈ JARVIS Arc Reactor v{VERSION} starting on {HOST}:{PORT}") await get_pool() await db_execute("UPDATE arc_status SET started_at=NOW(), last_heartbeat=NOW(), version=%s WHERE id=1", (VERSION,)) - await db_execute("""CREATE TABLE IF NOT EXISTS guardian_config ( - id INT AUTO_INCREMENT PRIMARY KEY, - key_name VARCHAR(64) NOT NULL, - value VARCHAR(255) NOT NULL DEFAULT '', - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE KEY uk_key (key_name) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") - await db_execute("""CREATE TABLE IF NOT EXISTS guardian_events ( - id INT AUTO_INCREMENT PRIMARY KEY, - event_type VARCHAR(64) NOT NULL, - severity ENUM('info','warning','critical') NOT NULL DEFAULT 'info', - agent_id VARCHAR(64) NOT NULL DEFAULT '', - hostname VARCHAR(128) NOT NULL DEFAULT '', - metric VARCHAR(64) NOT NULL DEFAULT '', - value FLOAT NOT NULL DEFAULT 0, - threshold FLOAT NOT NULL DEFAULT 0, - message TEXT NOT NULL, - ai_analysis TEXT NOT NULL DEFAULT '', - acknowledged TINYINT(1) NOT NULL DEFAULT 0, - created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - KEY idx_severity (severity), - KEY idx_ack (acknowledged), - KEY idx_created (created_at), - KEY idx_agent (agent_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") asyncio.create_task(job_poller()) asyncio.create_task(heartbeat_loop()) asyncio.create_task(guardian_loop()) @@ -2784,13 +2744,13 @@ async def guardian_chat_events(since: str = ""): """Return proactive guardian messages injected into conversations.""" if since: rows = await db_fetchall( - "SELECT id, content AS message, created_at FROM conversations " + "SELECT id, message, created_at FROM conversations " "WHERE session_id='guardian' AND created_at > %s ORDER BY created_at ASC", (since,) ) else: rows = await db_fetchall( - "SELECT id, content AS message, created_at FROM conversations " + "SELECT id, message, created_at FROM conversations " "WHERE session_id='guardian' ORDER BY created_at DESC LIMIT 10" ) return rows or []