diff --git a/api/endpoints/kb_intent_generator.php b/api/endpoints/kb_intent_generator.php new file mode 100644 index 0000000..0b562f2 --- /dev/null +++ b/api/endpoints/kb_intent_generator.php @@ -0,0 +1,334 @@ + 0) { + log_line(" Retry {$attempt}/{$retries} after rate-limit pause..."); + sleep(25); + } + $ch = curl_init('https://api.groq.com/openai/v1/chat/completions'); + curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_POST => true, + CURLOPT_TIMEOUT => 60, + CURLOPT_HTTPHEADER => [ + 'Authorization: Bearer ' . GROQ_API_KEY, + 'Content-Type: application/json', + ], + CURLOPT_POSTFIELDS => json_encode([ + 'model' => 'llama-3.3-70b-versatile', + 'max_tokens' => $max, + 'temperature' => 0.7, + 'messages' => [ + ['role' => 'system', 'content' => $system], + ['role' => 'user', 'content' => $user], + ], + ]), + ]); + $raw = curl_exec($ch); + $err = curl_error($ch); + $info = curl_getinfo($ch); + curl_close($ch); + + if ($err || !$raw) continue; + + // Check for rate limit response (429) + if (($info['http_code'] ?? 0) === 429) continue; + + $d = json_decode($raw, true); + $content = $d['choices'][0]['message']['content'] ?? null; + if ($content !== null) return $content; + } + return null; +} + +/* ── normalize a pattern to valid PHP PCRE ── */ +function normalize_pattern(string $pat): string { + $pat = trim($pat); + // If pattern already has PCRE delimiters, leave it alone + if (preg_match('/^[\/|~#!@%]/', $pat)) return $pat; + // Strip any (?i) inline flag — we'll add /i at the delimiter level + $pat = preg_replace('/^\(\?i\)/', '', $pat); + // Escape forward slashes inside the pattern + $pat = str_replace('/', '\\/', $pat); + return '/' . $pat . '/i'; +} + +/* ── daily guard: skip if already ran within last 20 hours ── */ +/* Set JARVIS_FORCE_RUN=1 (env) or pass --force (argv) to bypass */ +$lastRun = JarvisDB::single( + "SELECT updated_at FROM kb_facts WHERE category='kb_generator' AND fact_key='last_run'" +); +$forceRun = !empty(getenv('JARVIS_FORCE_RUN')) || (isset($argv[1]) && $argv[1] === '--force'); +if (!$forceRun && $lastRun && (time() - strtotime($lastRun['updated_at'])) < 72000) { + log_line('Skipping – ran within last 20 hours (next run tomorrow 3am). Use --force to override.'); + exit(0); +} +if ($forceRun) log_line('Force-run flag set — bypassing 20-hour guard.'); + +log_line('Starting daily KB intent generation run.'); + +/* ── topic batches (25 topics × ~40 intents = 1,000+) ── */ +$BATCHES = [ + ['id' => 'math_elem', 'category' => 'math_elementary', 'topic' => 'Elementary school mathematics', + 'desc' => 'counting, basic arithmetic, place value, simple fractions, 2D/3D shapes, measurement, telling time, money, patterns'], + ['id' => 'math_mid', 'category' => 'math_middle', 'topic' => 'Middle school mathematics', + 'desc' => 'ratios, percentages, integers, exponents, pre-algebra equations, coordinate planes, probability, statistics (mean/median/mode), geometry area/volume'], + ['id' => 'math_high', 'category' => 'math_high', 'topic' => 'High school mathematics', + 'desc' => 'quadratic equations, polynomials, functions, logarithms, trigonometry (SOH-CAH-TOA, unit circle), sequences, permutations and combinations, matrices'], + ['id' => 'math_college', 'category' => 'math_college', 'topic' => 'College-level mathematics', + 'desc' => 'limits and continuity, derivatives, integrals, chain rule, L\'Hopital, differential equations, vectors, eigenvalues, hypothesis testing, normal distribution'], + ['id' => 'bio_cell', 'category' => 'biology', 'topic' => 'Cell biology and genetics', + 'desc' => 'organelles, mitosis vs meiosis, DNA structure, protein synthesis (transcription/translation), Mendelian genetics, dominant/recessive, mutations, genetic disorders'], + ['id' => 'bio_body', 'category' => 'biology', 'topic' => 'Human body systems', + 'desc' => 'skeletal, muscular, cardiovascular, respiratory, digestive, nervous, endocrine, immune, reproductive, and excretory systems'], + ['id' => 'bio_ecology', 'category' => 'biology', 'topic' => 'Ecology and evolution', + 'desc' => 'ecosystems, biomes, food webs, trophic levels, symbiosis (mutualism/commensalism/parasitism), natural selection, adaptation, speciation, biodiversity'], + ['id' => 'chem_basics', 'category' => 'chemistry', 'topic' => 'Chemistry fundamentals', + 'desc' => 'atomic structure, periodic table trends, ionic vs covalent bonds, Lewis structures, polarity, molecular geometry (VSEPR), intermolecular forces'], + ['id' => 'chem_rxns', 'category' => 'chemistry', 'topic' => 'Chemical reactions and thermochemistry', + 'desc' => 'balancing equations, stoichiometry, limiting reagents, reaction types, enthalpy, entropy, Gibbs free energy, Le Chatelier\'s principle, equilibrium constants'], + ['id' => 'phys_mech', 'category' => 'physics', 'topic' => 'Physics – mechanics and energy', + 'desc' => 'kinematics, Newton\'s three laws, friction, momentum, conservation of energy, work and power, circular motion, gravitation, projectile motion'], + ['id' => 'phys_em', 'category' => 'physics', 'topic' => 'Physics – waves, electricity and magnetism', + 'desc' => 'wave properties (amplitude, frequency, wavelength), sound, light spectrum, reflection/refraction, Ohm\'s law, series vs parallel circuits, magnetic fields, electromagnetic induction'], + ['id' => 'earth_sci', 'category' => 'earth_science', 'topic' => 'Earth and environmental science', + 'desc' => 'rock cycle, plate tectonics, earthquakes, volcanoes, atmosphere layers, weather vs climate, greenhouse effect, water cycle, ocean currents, soil formation'], + ['id' => 'astronomy', 'category' => 'astronomy', 'topic' => 'Astronomy and space science', + 'desc' => 'solar system planets, star life cycles, galaxies, Big Bang theory, light-years, telescopes, space exploration milestones, dark matter/energy, black holes'], + ['id' => 'hist_us', 'category' => 'history_us', 'topic' => 'United States history', + 'desc' => 'colonial era, American Revolution, Constitution, westward expansion, Civil War, Reconstruction, Gilded Age, WWI, Great Depression, WWII, Civil Rights, Cold War, modern era'], + ['id' => 'hist_world', 'category' => 'history_world', 'topic' => 'World history', + 'desc' => 'ancient civilisations (Egypt, Greece, Rome, Mesopotamia, China, India), Medieval period, Renaissance, Reformation, age of exploration, colonialism, Industrial Revolution, WWI, WWII, decolonisation, Cold War'], + ['id' => 'geo_world', 'category' => 'geography', 'topic' => 'World geography', + 'desc' => 'continents and oceans, countries and capitals, physical features (mountains, rivers, deserts), climate zones, latitude/longitude, map projections, time zones, population distribution'], + ['id' => 'civics', 'category' => 'civics', 'topic' => 'Civics and US government', + 'desc' => 'three branches of government, checks and balances, Bill of Rights, Constitutional amendments, federalism, Electoral College, legislative process, Supreme Court landmark cases, political parties'], + ['id' => 'econ', 'category' => 'economics', 'topic' => 'Economics', + 'desc' => 'supply and demand, market equilibrium, elasticity, GDP, inflation, unemployment, fiscal vs monetary policy, comparative advantage, market structures (monopoly, oligopoly, perfect competition), opportunity cost'], + ['id' => 'lit_writing', 'category' => 'literature', 'topic' => 'Literature and writing', + 'desc' => 'figurative language (simile, metaphor, personification, irony), plot structure, literary themes, point of view, genre types, essay structure, thesis statements, grammar rules, poetry forms, citing sources'], + ['id' => 'cs_prog', 'category' => 'computer_science', 'topic' => 'Computer science and programming', + 'desc' => 'variables, data types, loops, conditionals, functions, OOP concepts, arrays/lists, sorting and searching algorithms, time/space complexity, binary, hexadecimal, recursion, debugging'], + ['id' => 'cs_systems', 'category' => 'computer_science', 'topic' => 'Computer systems and networking', + 'desc' => 'CPU/RAM/storage, operating systems, file systems, TCP/IP, DNS, HTTP/HTTPS, databases (SQL vs NoSQL), cybersecurity threats (phishing, SQL injection, XSS, malware), encryption basics, cloud computing'], + ['id' => 'psych', 'category' => 'psychology', 'topic' => 'Psychology', + 'desc' => 'classical and operant conditioning, Maslow\'s hierarchy, Piaget\'s stages, Erikson\'s stages, memory types, sleep stages, cognitive biases, Freud\'s theory, social influence, abnormal psychology basics'], + ['id' => 'phil', 'category' => 'philosophy', 'topic' => 'Philosophy and logic', + 'desc' => 'Socrates/Plato/Aristotle, ethical theories (utilitarianism, deontology, virtue ethics), epistemology, deductive vs inductive reasoning, logical fallacies, existentialism, free will vs determinism, political philosophy'], + ['id' => 'health_sci', 'category' => 'health', 'topic' => 'Health and life skills', + 'desc' => 'nutrition (macronutrients, vitamins, minerals), exercise physiology, mental health (anxiety, depression, stress response), reproductive health, substance abuse, first aid, disease prevention, sleep hygiene'], + ['id' => 'arts_music', 'category' => 'arts', 'topic' => 'Arts and music', + 'desc' => 'elements of art (line, shape, color, texture), colour theory, major art movements, famous artists and their works, music notes and scales, rhythm and meter, instrument families, major composers and genres'], +]; + +/* ── system prompt ── */ +$SYSTEM = <<<'SYS' +You are an expert educator generating KB (knowledge-base) intents for an AI assistant called JARVIS. +Each intent is a question/phrase a student might ask, paired with a clear educational answer. + +Respond ONLY with a valid JSON array (no markdown, no backticks, no commentary). +Each element must have exactly these keys: + "n" – intent_name: unique snake_case identifier ≤ 60 chars, prefixed with the batch id given + "p" – pattern: a raw regex string (NO delimiters, NO (?i) flag) that matches the question using \b word boundaries + "r" – response: a thorough but concise educational answer (2–5 sentences or a short structured list) + "c" – category: the category string provided + +Rules: +- Patterns must use \b word boundaries; escape backslashes for JSON (\\b not \b) +- Do NOT include regex delimiters (no leading / or trailing /i) — just the raw pattern +- Patterns should NOT start with ^ or end with $ +- Responses must be factually accurate +- Do not duplicate intent names; every "n" must be unique within this batch +- Return exactly 40 intents +SYS; + +/* ── insert helper ── */ +$inserted = 0; +$skipped = 0; +$errors = 0; + +function safe_insert(array $intent, string $batchCategory): void { + global $inserted, $skipped, $errors; + $name = trim($intent['n'] ?? ''); + $pattern = trim($intent['p'] ?? ''); + $response = trim($intent['r'] ?? ''); + $category = trim($intent['c'] ?? $batchCategory); + + if (!$name || !$pattern || !$response) { $errors++; return; } + if (strlen($name) > 64) $name = substr($name, 0, 64); + if (strlen($response) < 30) { $skipped++; return; } + + // Normalize to valid PHP PCRE with delimiters + $pattern = normalize_pattern($pattern); + if (strlen($pattern) > 512) $pattern = substr($pattern, 0, 511) . '/i'; + + // Validate before inserting — skip truly broken patterns + if (@preg_match($pattern, '') === false) { $errors++; return; } + + try { + JarvisDB::execute( + 'INSERT INTO kb_intents (intent_name, pattern, response_template, fact_category, action_type, priority, active) + VALUES (?, ?, ?, ?, ?, ?, 1) + ON DUPLICATE KEY UPDATE + pattern=VALUES(pattern), + response_template=VALUES(response_template), + fact_category=VALUES(fact_category), + active=1', + [$name, $pattern, $response, $category, 'response', 5] + ); + $inserted++; + } catch (Exception $e) { + $errors++; + } +} + +/* ── main generation loop ── */ +$totalBatches = count($BATCHES); +foreach ($BATCHES as $idx => $batch) { + $num = $idx + 1; + log_line("Batch {$num}/{$totalBatches}: {$batch['topic']}"); + + $user = "Generate 40 KB intents for the topic: {$batch['topic']}.\n" + . "Subtopics to cover: {$batch['desc']}.\n" + . "Prefix every intent_name with \"{$batch['id']}_\".\n" + . "Category string to use: \"{$batch['category']}\"."; + + $raw = groq($SYSTEM, $user, 3500); + if ($raw === null) { + log_line(" ✗ API call failed after retries – skipping batch."); + $errors += 40; + sleep(8); + continue; + } + + // Strip markdown code fences if model added them + $raw = preg_replace('/^```(?:json)?\s*/m', '', $raw); + $raw = preg_replace('/^```\s*/m', '', $raw); + $raw = trim($raw); + + // Extract JSON array (even if the model added preamble text) + if (!preg_match('/\[\s*\{.*\}\s*\]/s', $raw, $m)) { + log_line(" ✗ No JSON array found in response – skipping batch."); + $errors += 40; + sleep(8); + continue; + } + $items = json_decode($m[0], true); + if (!is_array($items)) { + log_line(" ✗ JSON parse failed – skipping batch."); + $errors += 40; + sleep(8); + continue; + } + + $batchInserted = 0; + foreach ($items as $item) { + if (!is_array($item)) continue; + safe_insert($item, $batch['category']); + $batchInserted++; + } + log_line(" ✓ Parsed {$batchInserted} intents (running total inserted: {$inserted})."); + + // Polite delay between API calls — Groq TPM limit needs ~8s between batches + if ($num < $totalBatches) sleep(8); +} + +log_line("Generation complete. Inserted/updated: {$inserted} | Short/invalid skipped: {$skipped} | Errors: {$errors}"); + +/* ── cleanup phase ── */ +log_line('Starting cleanup phase...'); + +// 1. Remove exact duplicate intent_names (keep the one with the longer response) +$dups = JarvisDB::query( + 'SELECT intent_name, COUNT(*) AS cnt FROM kb_intents GROUP BY intent_name HAVING cnt > 1' +); +$dupsPruned = 0; +foreach ($dups as $dup) { + $rows = JarvisDB::query( + 'SELECT id, LENGTH(response_template) AS rlen FROM kb_intents WHERE intent_name=? ORDER BY rlen DESC', + [$dup['intent_name']] + ); + array_shift($rows); + foreach ($rows as $row) { + JarvisDB::execute('DELETE FROM kb_intents WHERE id=?', [$row['id']]); + $dupsPruned++; + } +} +log_line(" Duplicate intent_names pruned: {$dupsPruned}"); + +// 2. Remove intents with very short responses (< 40 chars) +$shortPruned = JarvisDB::execute( + "DELETE FROM kb_intents WHERE LENGTH(response_template) < 40 AND priority <= 5" +); +log_line(" Short-response rows pruned: {$shortPruned}"); + +// 3. Trim whitespace on all generated intents +JarvisDB::execute( + "UPDATE kb_intents SET + intent_name = TRIM(intent_name), + pattern = TRIM(pattern), + response_template = TRIM(response_template), + fact_category = TRIM(fact_category) + WHERE priority = 5" +); +log_line(' Whitespace trimmed on all generated intents.'); + +// 4. Fix and validate PCRE patterns — normalize then deactivate only truly broken ones +$all = JarvisDB::query('SELECT id, pattern FROM kb_intents WHERE priority=5'); +$badPattern = 0; +$fixedPattern = 0; +foreach ($all as $row) { + $pat = normalize_pattern($row['pattern']); + if ($pat !== $row['pattern']) { + // Update to normalized form + JarvisDB::execute('UPDATE kb_intents SET pattern=?, active=1 WHERE id=?', [$pat, $row['id']]); + $fixedPattern++; + } elseif (@preg_match($pat, '') === false) { + JarvisDB::execute('UPDATE kb_intents SET active=0 WHERE id=?', [$row['id']]); + $badPattern++; + } else { + // Valid pattern — make sure it's active + JarvisDB::execute('UPDATE kb_intents SET active=1 WHERE id=?', [$row['id']]); + } +} +log_line(" Patterns normalized: {$fixedPattern} | Bad PCRE deactivated: {$badPattern}"); + +// 5. Enable ALL remaining inactive intents (including pre-existing ones) +$reactivated = JarvisDB::execute('UPDATE kb_intents SET active=1 WHERE active=0 AND priority <= 5'); +log_line(" Re-activated previously inactive intents: {$reactivated}"); + +// 6. Final stats +$stats = JarvisDB::single('SELECT COUNT(*) AS total, SUM(active) AS active FROM kb_intents'); +log_line("Final KB Intents table: {$stats['total']} total, {$stats['active']} active."); + +/* ── record last-run timestamp ── */ +JarvisDB::execute( + "INSERT INTO kb_facts (category, fact_key, fact_value, host) + VALUES ('kb_generator', 'last_run', NOW(), 'local') + ON DUPLICATE KEY UPDATE fact_value=NOW(), updated_at=NOW()", + [] +); +JarvisDB::execute( + "INSERT INTO kb_facts (category, fact_key, fact_value, host) + VALUES ('kb_generator', 'last_inserted', ?, 'local') + ON DUPLICATE KEY UPDATE fact_value=VALUES(fact_value), updated_at=NOW()", + [$inserted] +); + +log_line('Done.'); diff --git a/public_html/admin/index.php b/public_html/admin/index.php index 5ce080c..711e949 100644 --- a/public_html/admin/index.php +++ b/public_html/admin/index.php @@ -543,8 +543,9 @@ if ($action) { ]; if (isset($scripts[$wId])) { [$isPhp,$path] = $scripts[$wId]; + $env = ($wId === 'kb_intent_generator') ? 'JARVIS_FORCE_RUN=1 ' : ''; $cmd = $isPhp - ? '/usr/bin/php8.3 '.escapeshellarg($path).' >> /var/log/jarvis/cron.log 2>&1 &' + ? $env.'/usr/bin/php8.3 '.escapeshellarg($path).' >> /var/log/jarvis/cron.log 2>&1 &' : escapeshellcmd($path).' >> /var/log/jarvis/deploy.log 2>&1 &'; shell_exec($cmd); j(['ok'=>true,'msg'=>ucwords(str_replace('_',' ',$wId)).' triggered']); @@ -572,6 +573,32 @@ if ($action) { j(['ok'=>true,'version'=>$ag['version']??null,'status'=>$ag['status']??'unknown']); } else { bad('Invalid worker action'); } break; + case 'intent_gen_history': + $logFile = '/var/log/jarvis/cron.log'; + $result = ['last_success'=>null,'last_success_count'=>null,'failures'=>[]]; + if (file_exists($logFile)) { + $lines = file($logFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $cutoff = time() - 7 * 86400; + foreach (array_reverse($lines) as $line) { + if (stripos($line, 'KB Intent Generator') === false) continue; + // parse timestamp + preg_match('/^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]/', $line, $tm); + $ts = isset($tm[1]) ? strtotime($tm[1]) : 0; + // last success (done/complete) + if (!$result['last_success'] && preg_match('/done|complete/i', $line)) { + $result['last_success'] = $tm[1] ?? null; + preg_match('/inserted[:\s]+(\d+)/i', $line, $cnt); + $result['last_success_count'] = isset($cnt[1]) ? (int)$cnt[1] : null; + } + // failures in last 7 days + if ($ts >= $cutoff && preg_match('/error|fail/i', $line)) { + $result['failures'][] = $line; + if (count($result['failures']) >= 20) break; + } + } + $result['failures'] = array_reverse($result['failures']); + } + j($result); case 'intent_gen_log': $logFile = '/var/log/jarvis/cron.log'; $since = max(0, (int)($_GET['since'] ?? 0)); @@ -2233,31 +2260,61 @@ function wToast(msg,err=false) { setTimeout(()=>{t.style.opacity='0';},3000); } async function runIntentGenerator() { - // Open the working popup const modalHtml = `