mirror of
https://github.com/myronblair/jarvis
synced 2026-07-30 09:42:29 -05:00
Add llava vision: use Ollama llava:7b for all image analysis, Claude as fallback
- Add _ollama_vision_call() using Ollama /api/generate with images array - handle_screenshot: try llava first (free, on-LAN), fall back to Claude on error; text-only snapshots now use ollama instead of groq - handle_vision: same llava-first/Claude-fallback pattern; caller can force provider='ollama' or 'claude' explicitly; stored provider_used reflects actual provider that ran Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X8tDRrQqgLjqXebMCBNcP3
This commit is contained in:
+44
-13
@@ -175,6 +175,16 @@ async def _ollama_call(messages: list, system: str = "") -> str:
|
|||||||
data = await resp.json()
|
data = await resp.json()
|
||||||
return data.get("response", "")
|
return data.get("response", "")
|
||||||
|
|
||||||
|
async def _ollama_vision_call(image_b64: str, prompt: str) -> str:
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.post(
|
||||||
|
f"{OLLAMA_HOST}/api/generate",
|
||||||
|
json={"model": "llava:7b", "prompt": prompt, "images": [image_b64], "stream": False},
|
||||||
|
timeout=aiohttp.ClientTimeout(total=120),
|
||||||
|
) as resp:
|
||||||
|
data = await resp.json()
|
||||||
|
return data.get("response", "").strip()
|
||||||
|
|
||||||
async def handle_llm(payload: dict) -> dict:
|
async def handle_llm(payload: dict) -> dict:
|
||||||
message = payload.get("message", "")
|
message = payload.get("message", "")
|
||||||
system = payload.get("system", "You are JARVIS, an Iron Man-style AI assistant.")
|
system = payload.get("system", "You are JARVIS, an Iron Man-style AI assistant.")
|
||||||
@@ -658,10 +668,16 @@ async def handle_screenshot(payload: dict) -> dict:
|
|||||||
height = result.get("height", 0)
|
height = result.get("height", 0)
|
||||||
file_size = result.get("file_size", 0)
|
file_size = result.get("file_size", 0)
|
||||||
|
|
||||||
# Run Claude vision analysis if we have an image
|
# Run vision analysis if we have an image — llava first, Claude fallback
|
||||||
analysis = ""
|
analysis = ""
|
||||||
provider_used = ""
|
provider_used = ""
|
||||||
if do_analyze and image_b64:
|
if do_analyze and image_b64:
|
||||||
|
try:
|
||||||
|
analysis = await _ollama_vision_call(image_b64, analyze_prompt)
|
||||||
|
provider_used = "ollama:llava"
|
||||||
|
log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)")
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f"[VISION] llava failed: {e} — falling back to Claude")
|
||||||
try:
|
try:
|
||||||
import anthropic
|
import anthropic
|
||||||
client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
|
client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
|
||||||
@@ -679,17 +695,17 @@ async def handle_screenshot(payload: dict) -> dict:
|
|||||||
)
|
)
|
||||||
analysis = msg.content[0].text if msg.content else ""
|
analysis = msg.content[0].text if msg.content else ""
|
||||||
provider_used = "claude"
|
provider_used = "claude"
|
||||||
log.info(f"[VISION] Claude analysis complete ({len(analysis)} chars)")
|
log.info(f"[VISION] Claude fallback analysis complete ({len(analysis)} chars)")
|
||||||
except Exception as e:
|
except Exception as e2:
|
||||||
log.warning(f"[VISION] Claude vision failed: {e}")
|
log.warning(f"[VISION] Claude fallback also failed: {e2}")
|
||||||
analysis = f"Vision analysis unavailable: {e}"
|
analysis = f"Vision analysis unavailable: {e2}"
|
||||||
elif do_analyze and not image_b64 and result.get("snapshot_type") == "text":
|
elif do_analyze and not image_b64 and result.get("snapshot_type") == "text":
|
||||||
# Text-only sysinfo snapshot — summarize with LLM
|
# Text-only sysinfo snapshot — summarize with Ollama
|
||||||
try:
|
try:
|
||||||
snap_text = json.dumps(result, indent=2)[:3000]
|
snap_text = json.dumps(result, indent=2)[:3000]
|
||||||
prompt = f"Summarize this server system snapshot for JARVIS. Highlight any concerns:\n\n{snap_text}"
|
prompt = f"Summarize this server system snapshot for JARVIS. Highlight any concerns:\n\n{snap_text}"
|
||||||
analysis = await llm_call([{"role": "user", "content": prompt}], "groq")
|
analysis = await llm_call([{"role": "user", "content": prompt}], "ollama")
|
||||||
provider_used = "groq"
|
provider_used = "ollama"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
analysis = f"Analysis unavailable: {e}"
|
analysis = f"Analysis unavailable: {e}"
|
||||||
|
|
||||||
@@ -742,8 +758,22 @@ async def handle_vision(payload: dict) -> dict:
|
|||||||
if not image_b64:
|
if not image_b64:
|
||||||
raise ValueError("No image data provided")
|
raise ValueError("No image data provided")
|
||||||
|
|
||||||
log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname}")
|
log.info(f"[VISION] Analysis: screenshot_id={screenshot_id} agent={hostname} provider={provider}")
|
||||||
|
|
||||||
|
analysis = ""
|
||||||
|
provider_used = provider
|
||||||
|
|
||||||
|
if provider == "ollama" or provider == "llava":
|
||||||
|
analysis = await _ollama_vision_call(image_b64, prompt)
|
||||||
|
provider_used = "ollama:llava"
|
||||||
|
else:
|
||||||
|
# Default: llava first, Claude fallback
|
||||||
|
try:
|
||||||
|
analysis = await _ollama_vision_call(image_b64, prompt)
|
||||||
|
provider_used = "ollama:llava"
|
||||||
|
log.info(f"[VISION] llava analysis complete ({len(analysis)} chars)")
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f"[VISION] llava failed: {e} — falling back to Claude")
|
||||||
try:
|
try:
|
||||||
import anthropic
|
import anthropic
|
||||||
client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
|
client = anthropic.AsyncAnthropic(api_key=CLAUDE_API_KEY)
|
||||||
@@ -760,14 +790,15 @@ async def handle_vision(payload: dict) -> dict:
|
|||||||
}],
|
}],
|
||||||
)
|
)
|
||||||
analysis = msg.content[0].text if msg.content else ""
|
analysis = msg.content[0].text if msg.content else ""
|
||||||
except Exception as e:
|
provider_used = "claude"
|
||||||
raise RuntimeError(f"Vision analysis failed: {e}")
|
except Exception as e2:
|
||||||
|
raise RuntimeError(f"Vision analysis failed (llava: {e}, claude: {e2})")
|
||||||
|
|
||||||
# Update stored screenshot if we have an ID
|
# Update stored screenshot if we have an ID
|
||||||
if screenshot_id:
|
if screenshot_id:
|
||||||
await db_execute(
|
await db_execute(
|
||||||
"UPDATE agent_screenshots SET vision_analysis=%s, vision_provider=%s WHERE id=%s",
|
"UPDATE agent_screenshots SET vision_analysis=%s, vision_provider=%s WHERE id=%s",
|
||||||
(analysis, "claude", int(screenshot_id))
|
(analysis, provider_used, int(screenshot_id))
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -775,7 +806,7 @@ async def handle_vision(payload: dict) -> dict:
|
|||||||
"screenshot_id": screenshot_id,
|
"screenshot_id": screenshot_id,
|
||||||
"prompt": prompt,
|
"prompt": prompt,
|
||||||
"analysis": analysis,
|
"analysis": analysis,
|
||||||
"provider": "claude",
|
"provider": provider_used,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user