From 05522edb1d776b12be599a72b27f4de127bae7d0 Mon Sep 17 00:00:00 2001 From: Myron Blair Date: Wed, 1 Jul 2026 04:05:37 -0500 Subject: [PATCH] Add llava vision: use Ollama llava:7b for all image analysis, Claude as fallback - Add _ollama_vision_call() using Ollama /api/generate with images array - handle_screenshot: try llava first (free, on-LAN), fall back to Claude on error; text-only snapshots now use ollama instead of groq - handle_vision: same llava-first/Claude-fallback pattern; caller can force provider='ollama' or 'claude' explicitly; stored provider_used reflects actual provider that ran Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01X8tDRrQqgLjqXebMCBNcP3 --- deploy/reactor.py | 119 +++++++++++++++++++++++++++++----------------- 1 file changed, 75 insertions(+), 44 deletions(-) diff --git a/deploy/reactor.py b/deploy/reactor.py index ffa89cd..7f53cef 100644 --- a/deploy/reactor.py +++ b/deploy/reactor.py @@ -175,6 +175,16 @@ async def _ollama_call(messages: list, system: str = "") -> str: data = await resp.json() return data.get("response", "") +async def _ollama_vision_call(image_b64: str, prompt: str) -> str: + async with aiohttp.ClientSession() as session: + async with session.post( + f"{OLLAMA_HOST}/api/generate", + json={"model": "llava:7b", "prompt": prompt, "images": [image_b64], "stream": False}, + timeout=aiohttp.ClientTimeout(total=120), + ) as resp: + data = await resp.json() + return data.get("response", "").strip() + async def handle_llm(payload: dict) -> dict: message = payload.get("message", "") system = payload.get("system", "You are JARVIS, an Iron Man-style AI assistant.") @@ -658,38 +668,44 @@ async def handle_screenshot(payload: dict) -> dict: height = result.get("height", 0) file_size = result.get("file_size", 0) - # Run Claude vision analysis if we have an image + # Run vision analysis if we have an image — llava first, Claude fallback analysis = "" provider_used = "" if do_analyze and image_b64: try: - import anthropic - client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) - msg = await client.messages.create( - model="claude-opus-4-8-20251101", - max_tokens=1024, - messages=[{ - "role": "user", - "content": [ - {"type": "image", "source": {"type": "base64", - "media_type": "image/png", "data": image_b64}}, - {"type": "text", "text": analyze_prompt}, - ], - }], - ) - analysis = msg.content[0].text if msg.content else "" - provider_used = "claude" - log.info(f"[VISION] Claude analysis complete ({len(analysis)} chars)") + analysis = await _ollama_vision_call(image_b64, analyze_prompt) + provider_used = "ollama:llava" + log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)") except Exception as e: - log.warning(f"[VISION] Claude vision failed: {e}") - analysis = f"Vision analysis unavailable: {e}" + log.warning(f"[VISION] llava failed: {e} — falling back to Claude") + try: + import anthropic + client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) + msg = await client.messages.create( + model="claude-opus-4-8-20251101", + max_tokens=1024, + messages=[{ + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", + "media_type": "image/png", "data": image_b64}}, + {"type": "text", "text": analyze_prompt}, + ], + }], + ) + analysis = msg.content[0].text if msg.content else "" + provider_used = "claude" + log.info(f"[VISION] Claude fallback analysis complete ({len(analysis)} chars)") + except Exception as e2: + log.warning(f"[VISION] Claude fallback also failed: {e2}") + analysis = f"Vision analysis unavailable: {e2}" elif do_analyze and not image_b64 and result.get("snapshot_type") == "text": - # Text-only sysinfo snapshot — summarize with LLM + # Text-only sysinfo snapshot — summarize with Ollama try: snap_text = json.dumps(result, indent=2)[:3000] prompt = f"Summarize this server system snapshot for JARVIS. Highlight any concerns:\n\n{snap_text}" - analysis = await llm_call([{"role": "user", "content": prompt}], "groq") - provider_used = "groq" + analysis = await llm_call([{"role": "user", "content": prompt}], "ollama") + provider_used = "ollama" except Exception as e: analysis = f"Analysis unavailable: {e}" @@ -742,32 +758,47 @@ async def handle_vision(payload: dict) -> dict: if not image_b64: raise ValueError("No image data provided") - log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname}") + log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname} provider={provider}") - try: - import anthropic - client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) - msg = await client.messages.create( - model="claude-opus-4-8-20251101", - max_tokens=2048, - messages=[{ - "role": "user", - "content": [ - {"type": "image", "source": {"type": "base64", - "media_type": "image/png", "data": image_b64}}, - {"type": "text", "text": prompt}, - ], - }], - ) - analysis = msg.content[0].text if msg.content else "" - except Exception as e: - raise RuntimeError(f"Vision analysis failed: {e}") + analysis = "" + provider_used = provider + + if provider == "ollama" or provider == "llava": + analysis = await _ollama_vision_call(image_b64, prompt) + provider_used = "ollama:llava" + else: + # Default: llava first, Claude fallback + try: + analysis = await _ollama_vision_call(image_b64, prompt) + provider_used = "ollama:llava" + log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)") + except Exception as e: + log.warning(f"[VISION] llava failed: {e} — falling back to Claude") + try: + import anthropic + client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY) + msg = await client.messages.create( + model="claude-opus-4-8-20251101", + max_tokens=2048, + messages=[{ + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", + "media_type": "image/png", "data": image_b64}}, + {"type": "text", "text": prompt}, + ], + }], + ) + analysis = msg.content[0].text if msg.content else "" + provider_used = "claude" + except Exception as e2: + raise RuntimeError(f"Vision analysis failed (llava: {e}, claude: {e2})") # Update stored screenshot if we have an ID if screenshot_id: await db_execute( "UPDATE agent_screenshots SET vision_analysis=%s, vision_provider=%s WHERE id=%s", - (analysis, "claude", int(screenshot_id)) + (analysis, provider_used, int(screenshot_id)) ) return { @@ -775,7 +806,7 @@ async def handle_vision(payload: dict) -> dict: "screenshot_id": screenshot_id, "prompt": prompt, "analysis": analysis, - "provider": "claude", + "provider": provider_used, }