mirror of
https://github.com/myronblair/jarvis
synced 2026-07-27 16:22:55 -05:00
Fix errors unmasked by re-enabling error_reporting(E_ALL)
- config.example.php: error_reporting(0) -> E_ALL (live config.php matches); add JELLYFIN_URL/JELLYFIN_API_KEY placeholders - history.php, jellyfin.php: drop require of nonexistent includes/auth.php + AuthMiddleware call (router enforces auth centrally) — both endpoints were fataling on every request - remove stale kb_intent_generator .bak files from deployed tree DB (not in repo): kb_facts.fact_value TEXT -> MEDIUMTEXT; ha/entity_map had failed every write since Jul 2 at >64KB Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -47,8 +47,11 @@ define(chr(39)+'HA_TOKEN'.chr(39), chr(39)+'YOUR_HA_LONG_LIVED_TOKEN'.chr(39));
|
|||||||
define(chr(39)+'SESSION_LIFETIME'.chr(39), 86400 * 7);
|
define(chr(39)+'SESSION_LIFETIME'.chr(39), 86400 * 7);
|
||||||
define(chr(39)+'SITE_URL'.chr(39), chr(39)+'https://jarvis.orbishosting.com'.chr(39));
|
define(chr(39)+'SITE_URL'.chr(39), chr(39)+'https://jarvis.orbishosting.com'.chr(39));
|
||||||
|
|
||||||
error_reporting(0);
|
error_reporting(E_ALL);
|
||||||
ini_set(chr(39)+'display_errors'.chr(39), 0);
|
ini_set(chr(39)+'display_errors'.chr(39), 0);
|
||||||
ini_set(chr(39)+'log_errors'.chr(39), 1);
|
ini_set(chr(39)+'log_errors'.chr(39), 1);
|
||||||
ini_set(chr(39)+'error_log'.chr(39), chr(39)+'/var/log/apache2/jarvis_errors.log'.chr(39));
|
ini_set(chr(39)+'error_log'.chr(39), chr(39)+'/var/log/apache2/jarvis_errors.log'.chr(39));
|
||||||
date_default_timezone_set(chr(39)+'America/Chicago'.chr(39));
|
date_default_timezone_set(chr(39)+'America/Chicago'.chr(39));
|
||||||
|
|
||||||
|
define('JELLYFIN_URL', 'http://10.48.200.33:8096');
|
||||||
|
define('JELLYFIN_API_KEY', 'your-jellyfin-api-key');
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
<?php
|
<?php
|
||||||
// Chat history search endpoint
|
// Chat history search endpoint
|
||||||
require_once __DIR__ . '/../config.php';
|
require_once __DIR__ . '/../config.php';
|
||||||
require_once __DIR__ . '/../../includes/auth.php';
|
|
||||||
|
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
AuthMiddleware::requireAuth();
|
|
||||||
|
|
||||||
$q = trim($_GET['q'] ?? '');
|
$q = trim($_GET['q'] ?? '');
|
||||||
if (strlen($q) < 2) {
|
if (strlen($q) < 2) {
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
<?php
|
<?php
|
||||||
require_once __DIR__ . '/../config.php';
|
require_once __DIR__ . '/../config.php';
|
||||||
require_once __DIR__ . '/../../includes/auth.php';
|
|
||||||
|
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
AuthMiddleware::requireAuth();
|
|
||||||
|
|
||||||
$action = $_GET['action'] ?? 'sessions';
|
$action = $_GET['action'] ?? 'sessions';
|
||||||
|
|
||||||
|
|||||||
@@ -1,284 +0,0 @@
|
|||||||
<?php
|
|
||||||
/**
|
|
||||||
* JARVIS KB Intent Generator
|
|
||||||
* Generates 1,000+ educational KB intents via Groq LLM and imports to kb_intents table.
|
|
||||||
* Also runs a cleanup pass: deduplication, short-response pruning, pattern normalisation.
|
|
||||||
*
|
|
||||||
* CLI / cron: /usr/bin/php8.3 /var/www/jarvis/api/endpoints/kb_intent_generator.php
|
|
||||||
* Schedule: Weekly – Sunday 3 am
|
|
||||||
*/
|
|
||||||
|
|
||||||
require_once __DIR__ . '/../config.php';
|
|
||||||
require_once __DIR__ . '/../lib/db.php';
|
|
||||||
|
|
||||||
/* ── helpers ── */
|
|
||||||
function ts(): string { return '[' . date('Y-m-d H:i:s') . ']'; }
|
|
||||||
function log_line(string $msg): void { echo ts() . ' KB Intent Generator: ' . $msg . "\n"; flush(); }
|
|
||||||
function groq(string $system, string $user, int $max = 3000): ?string {
|
|
||||||
$ch = curl_init('https://api.groq.com/openai/v1/chat/completions');
|
|
||||||
curl_setopt_array($ch, [
|
|
||||||
CURLOPT_RETURNTRANSFER => true,
|
|
||||||
CURLOPT_POST => true,
|
|
||||||
CURLOPT_TIMEOUT => 60,
|
|
||||||
CURLOPT_HTTPHEADER => [
|
|
||||||
'Authorization: Bearer ' . GROQ_API_KEY,
|
|
||||||
'Content-Type: application/json',
|
|
||||||
],
|
|
||||||
CURLOPT_POSTFIELDS => json_encode([
|
|
||||||
'model' => 'llama-3.3-70b-versatile',
|
|
||||||
'max_tokens' => $max,
|
|
||||||
'temperature' => 0.7,
|
|
||||||
'messages' => [
|
|
||||||
['role' => 'system', 'content' => $system],
|
|
||||||
['role' => 'user', 'content' => $user],
|
|
||||||
],
|
|
||||||
]),
|
|
||||||
]);
|
|
||||||
$raw = curl_exec($ch);
|
|
||||||
$err = curl_error($ch);
|
|
||||||
curl_close($ch);
|
|
||||||
if ($err || !$raw) return null;
|
|
||||||
$d = json_decode($raw, true);
|
|
||||||
return $d['choices'][0]['message']['content'] ?? null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ── daily guard: skip if already ran within last 20 hours ── */
|
|
||||||
$lastRun = JarvisDB::single(
|
|
||||||
"SELECT updated_at FROM kb_facts WHERE category='kb_generator' AND fact_key='last_run'"
|
|
||||||
);
|
|
||||||
if ($lastRun && (time() - strtotime($lastRun['updated_at'])) < 72000) {
|
|
||||||
log_line('Skipping – ran within last 6 days (next run Sunday 3am).');
|
|
||||||
exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
log_line('Starting weekly KB intent generation run.');
|
|
||||||
|
|
||||||
/* ── topic batches (25 topics × ~40 intents = 1,000+) ── */
|
|
||||||
$BATCHES = [
|
|
||||||
['id' => 'math_elem', 'category' => 'math_elementary', 'topic' => 'Elementary school mathematics',
|
|
||||||
'desc' => 'counting, basic arithmetic, place value, simple fractions, 2D/3D shapes, measurement, telling time, money, patterns'],
|
|
||||||
['id' => 'math_mid', 'category' => 'math_middle', 'topic' => 'Middle school mathematics',
|
|
||||||
'desc' => 'ratios, percentages, integers, exponents, pre-algebra equations, coordinate planes, probability, statistics (mean/median/mode), geometry area/volume'],
|
|
||||||
['id' => 'math_high', 'category' => 'math_high', 'topic' => 'High school mathematics',
|
|
||||||
'desc' => 'quadratic equations, polynomials, functions, logarithms, trigonometry (SOH-CAH-TOA, unit circle), sequences, permutations and combinations, matrices'],
|
|
||||||
['id' => 'math_college', 'category' => 'math_college', 'topic' => 'College-level mathematics',
|
|
||||||
'desc' => 'limits and continuity, derivatives, integrals, chain rule, L\'Hopital, differential equations, vectors, eigenvalues, hypothesis testing, normal distribution'],
|
|
||||||
['id' => 'bio_cell', 'category' => 'biology', 'topic' => 'Cell biology and genetics',
|
|
||||||
'desc' => 'organelles, mitosis vs meiosis, DNA structure, protein synthesis (transcription/translation), Mendelian genetics, dominant/recessive, mutations, genetic disorders'],
|
|
||||||
['id' => 'bio_body', 'category' => 'biology', 'topic' => 'Human body systems',
|
|
||||||
'desc' => 'skeletal, muscular, cardiovascular, respiratory, digestive, nervous, endocrine, immune, reproductive, and excretory systems'],
|
|
||||||
['id' => 'bio_ecology', 'category' => 'biology', 'topic' => 'Ecology and evolution',
|
|
||||||
'desc' => 'ecosystems, biomes, food webs, trophic levels, symbiosis (mutualism/commensalism/parasitism), natural selection, adaptation, speciation, biodiversity'],
|
|
||||||
['id' => 'chem_basics', 'category' => 'chemistry', 'topic' => 'Chemistry fundamentals',
|
|
||||||
'desc' => 'atomic structure, periodic table trends, ionic vs covalent bonds, Lewis structures, polarity, molecular geometry (VSEPR), intermolecular forces'],
|
|
||||||
['id' => 'chem_rxns', 'category' => 'chemistry', 'topic' => 'Chemical reactions and thermochemistry',
|
|
||||||
'desc' => 'balancing equations, stoichiometry, limiting reagents, reaction types, enthalpy, entropy, Gibbs free energy, Le Chatelier\'s principle, equilibrium constants'],
|
|
||||||
['id' => 'phys_mech', 'category' => 'physics', 'topic' => 'Physics – mechanics and energy',
|
|
||||||
'desc' => 'kinematics, Newton\'s three laws, friction, momentum, conservation of energy, work and power, circular motion, gravitation, projectile motion'],
|
|
||||||
['id' => 'phys_em', 'category' => 'physics', 'topic' => 'Physics – waves, electricity and magnetism',
|
|
||||||
'desc' => 'wave properties (amplitude, frequency, wavelength), sound, light spectrum, reflection/refraction, Ohm\'s law, series vs parallel circuits, magnetic fields, electromagnetic induction'],
|
|
||||||
['id' => 'earth_sci', 'category' => 'earth_science', 'topic' => 'Earth and environmental science',
|
|
||||||
'desc' => 'rock cycle, plate tectonics, earthquakes, volcanoes, atmosphere layers, weather vs climate, greenhouse effect, water cycle, ocean currents, soil formation'],
|
|
||||||
['id' => 'astronomy', 'category' => 'astronomy', 'topic' => 'Astronomy and space science',
|
|
||||||
'desc' => 'solar system planets, star life cycles, galaxies, Big Bang theory, light-years, telescopes, space exploration milestones, dark matter/energy, black holes'],
|
|
||||||
['id' => 'hist_us', 'category' => 'history_us', 'topic' => 'United States history',
|
|
||||||
'desc' => 'colonial era, American Revolution, Constitution, westward expansion, Civil War, Reconstruction, Gilded Age, WWI, Great Depression, WWII, Civil Rights, Cold War, modern era'],
|
|
||||||
['id' => 'hist_world', 'category' => 'history_world', 'topic' => 'World history',
|
|
||||||
'desc' => 'ancient civilisations (Egypt, Greece, Rome, Mesopotamia, China, India), Medieval period, Renaissance, Reformation, age of exploration, colonialism, Industrial Revolution, WWI, WWII, decolonisation, Cold War'],
|
|
||||||
['id' => 'geo_world', 'category' => 'geography', 'topic' => 'World geography',
|
|
||||||
'desc' => 'continents and oceans, countries and capitals, physical features (mountains, rivers, deserts), climate zones, latitude/longitude, map projections, time zones, population distribution'],
|
|
||||||
['id' => 'civics', 'category' => 'civics', 'topic' => 'Civics and US government',
|
|
||||||
'desc' => 'three branches of government, checks and balances, Bill of Rights, Constitutional amendments, federalism, Electoral College, legislative process, Supreme Court landmark cases, political parties'],
|
|
||||||
['id' => 'econ', 'category' => 'economics', 'topic' => 'Economics',
|
|
||||||
'desc' => 'supply and demand, market equilibrium, elasticity, GDP, inflation, unemployment, fiscal vs monetary policy, comparative advantage, market structures (monopoly, oligopoly, perfect competition), opportunity cost'],
|
|
||||||
['id' => 'lit_writing', 'category' => 'literature', 'topic' => 'Literature and writing',
|
|
||||||
'desc' => 'figurative language (simile, metaphor, personification, irony), plot structure, literary themes, point of view, genre types, essay structure, thesis statements, grammar rules, poetry forms, citing sources'],
|
|
||||||
['id' => 'cs_prog', 'category' => 'computer_science', 'topic' => 'Computer science and programming',
|
|
||||||
'desc' => 'variables, data types, loops, conditionals, functions, OOP concepts, arrays/lists, sorting and searching algorithms, time/space complexity, binary, hexadecimal, recursion, debugging'],
|
|
||||||
['id' => 'cs_systems', 'category' => 'computer_science', 'topic' => 'Computer systems and networking',
|
|
||||||
'desc' => 'CPU/RAM/storage, operating systems, file systems, TCP/IP, DNS, HTTP/HTTPS, databases (SQL vs NoSQL), cybersecurity threats (phishing, SQL injection, XSS, malware), encryption basics, cloud computing'],
|
|
||||||
['id' => 'psych', 'category' => 'psychology', 'topic' => 'Psychology',
|
|
||||||
'desc' => 'classical and operant conditioning, Maslow\'s hierarchy, Piaget\'s stages, Erikson\'s stages, memory types, sleep stages, cognitive biases, Freud\'s theory, social influence, abnormal psychology basics'],
|
|
||||||
['id' => 'phil', 'category' => 'philosophy', 'topic' => 'Philosophy and logic',
|
|
||||||
'desc' => 'Socrates/Plato/Aristotle, ethical theories (utilitarianism, deontology, virtue ethics), epistemology, deductive vs inductive reasoning, logical fallacies, existentialism, free will vs determinism, political philosophy'],
|
|
||||||
['id' => 'health_sci', 'category' => 'health', 'topic' => 'Health and life skills',
|
|
||||||
'desc' => 'nutrition (macronutrients, vitamins, minerals), exercise physiology, mental health (anxiety, depression, stress response), reproductive health, substance abuse, first aid, disease prevention, sleep hygiene'],
|
|
||||||
['id' => 'arts_music', 'category' => 'arts', 'topic' => 'Arts and music',
|
|
||||||
'desc' => 'elements of art (line, shape, color, texture), colour theory, major art movements, famous artists and their works, music notes and scales, rhythm and meter, instrument families, major composers and genres'],
|
|
||||||
];
|
|
||||||
|
|
||||||
/* ── system prompt ── */
|
|
||||||
$SYSTEM = <<<'SYS'
|
|
||||||
You are an expert educator generating KB (knowledge-base) intents for an AI assistant called JARVIS.
|
|
||||||
Each intent is a question/phrase a student might ask, paired with a clear educational answer.
|
|
||||||
|
|
||||||
Respond ONLY with a valid JSON array (no markdown, no backticks, no commentary).
|
|
||||||
Each element must have exactly these keys:
|
|
||||||
"n" – intent_name: unique snake_case identifier ≤ 60 chars, prefixed with the batch id given
|
|
||||||
"p" – pattern: a PHP PCRE regex (use (?i) for case-insensitive) that matches the question
|
|
||||||
"r" – response: a thorough but concise educational answer (2–5 sentences or a short structured list)
|
|
||||||
"c" – category: the category string provided
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
- Patterns must use \\b word boundaries; escape backslashes for JSON (\\b not \b)
|
|
||||||
- Patterns should NOT start with ^ or end with $ (they are substring matches)
|
|
||||||
- Responses must be factually accurate
|
|
||||||
- Do not duplicate intent names; every "n" must be unique within this batch
|
|
||||||
- Return exactly 40 intents
|
|
||||||
SYS;
|
|
||||||
|
|
||||||
/* ── insert helper ── */
|
|
||||||
$inserted = 0;
|
|
||||||
$skipped = 0;
|
|
||||||
$errors = 0;
|
|
||||||
|
|
||||||
function safe_insert(array $intent, string $batchCategory): void {
|
|
||||||
global $inserted, $skipped, $errors;
|
|
||||||
$name = trim($intent['n'] ?? '');
|
|
||||||
$pattern = trim($intent['p'] ?? '');
|
|
||||||
$response = trim($intent['r'] ?? '');
|
|
||||||
$category = trim($intent['c'] ?? $batchCategory);
|
|
||||||
|
|
||||||
if (!$name || !$pattern || !$response) { $errors++; return; }
|
|
||||||
if (strlen($name) > 64) $name = substr($name, 0, 64);
|
|
||||||
if (strlen($pattern) > 512) $pattern = substr($pattern, 0, 512);
|
|
||||||
if (strlen($response) < 30) { $skipped++; return; } // too short
|
|
||||||
|
|
||||||
try {
|
|
||||||
JarvisDB::execute(
|
|
||||||
'INSERT INTO kb_intents (intent_name, pattern, response_template, fact_category, action_type, priority, active)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
||||||
ON DUPLICATE KEY UPDATE
|
|
||||||
pattern=VALUES(pattern),
|
|
||||||
response_template=VALUES(response_template),
|
|
||||||
fact_category=VALUES(fact_category)',
|
|
||||||
[$name, $pattern, $response, $category, 'response', 5, 1]
|
|
||||||
);
|
|
||||||
$inserted++;
|
|
||||||
} catch (Exception $e) {
|
|
||||||
$errors++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ── main generation loop ── */
|
|
||||||
$totalBatches = count($BATCHES);
|
|
||||||
foreach ($BATCHES as $idx => $batch) {
|
|
||||||
$num = $idx + 1;
|
|
||||||
log_line("Batch {$num}/{$totalBatches}: {$batch['topic']}");
|
|
||||||
|
|
||||||
$user = "Generate 40 KB intents for the topic: {$batch['topic']}.\n"
|
|
||||||
. "Subtopics to cover: {$batch['desc']}.\n"
|
|
||||||
. "Prefix every intent_name with \"{$batch['id']}_\".\n"
|
|
||||||
. "Category string to use: \"{$batch['category']}\".";
|
|
||||||
|
|
||||||
$raw = groq($SYSTEM, $user, 3500);
|
|
||||||
if ($raw === null) {
|
|
||||||
log_line(" ✗ API call failed – skipping batch.");
|
|
||||||
$errors += 40;
|
|
||||||
sleep(3);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strip markdown code fences if model added them
|
|
||||||
$raw = preg_replace('/^```(?:json)?\s*/m', '', $raw);
|
|
||||||
$raw = preg_replace('/^```\s*/m', '', $raw);
|
|
||||||
$raw = trim($raw);
|
|
||||||
|
|
||||||
// Extract JSON array (even if the model added preamble text)
|
|
||||||
if (!preg_match('/\[\s*\{.*\}\s*\]/s', $raw, $m)) {
|
|
||||||
log_line(" ✗ No JSON array found in response – skipping batch.");
|
|
||||||
$errors += 40;
|
|
||||||
sleep(2);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$items = json_decode($m[0], true);
|
|
||||||
if (!is_array($items)) {
|
|
||||||
log_line(" ✗ JSON parse failed – skipping batch.");
|
|
||||||
$errors += 40;
|
|
||||||
sleep(2);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$batchInserted = 0;
|
|
||||||
foreach ($items as $item) {
|
|
||||||
if (!is_array($item)) continue;
|
|
||||||
safe_insert($item, $batch['category']);
|
|
||||||
$batchInserted++;
|
|
||||||
}
|
|
||||||
log_line(" ✓ Parsed {$batchInserted} intents (running total inserted: {$inserted}).");
|
|
||||||
|
|
||||||
// Polite delay between API calls to avoid rate limiting
|
|
||||||
if ($num < $totalBatches) sleep(2);
|
|
||||||
}
|
|
||||||
|
|
||||||
log_line("Generation complete. Inserted/updated: {$inserted} | Short/invalid skipped: {$skipped} | Errors: {$errors}");
|
|
||||||
|
|
||||||
/* ── cleanup phase ── */
|
|
||||||
log_line('Starting cleanup phase...');
|
|
||||||
|
|
||||||
// 1. Remove exact duplicate intent_names (keep the one with the longer response)
|
|
||||||
$dups = JarvisDB::query(
|
|
||||||
'SELECT intent_name, COUNT(*) AS cnt FROM kb_intents GROUP BY intent_name HAVING cnt > 1'
|
|
||||||
);
|
|
||||||
$dupsPruned = 0;
|
|
||||||
foreach ($dups as $dup) {
|
|
||||||
$rows = JarvisDB::query(
|
|
||||||
'SELECT id, LENGTH(response_template) AS rlen FROM kb_intents WHERE intent_name=? ORDER BY rlen DESC',
|
|
||||||
[$dup['intent_name']]
|
|
||||||
);
|
|
||||||
// Delete all but the first (longest response)
|
|
||||||
array_shift($rows);
|
|
||||||
foreach ($rows as $row) {
|
|
||||||
JarvisDB::execute('DELETE FROM kb_intents WHERE id=?', [$row['id']]);
|
|
||||||
$dupsPruned++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log_line(" Duplicate intent_names pruned: {$dupsPruned}");
|
|
||||||
|
|
||||||
// 2. Remove intents with very short responses (< 40 chars) — likely garbage
|
|
||||||
$shortPruned = JarvisDB::execute(
|
|
||||||
"DELETE FROM kb_intents WHERE LENGTH(response_template) < 40 AND priority <= 5"
|
|
||||||
);
|
|
||||||
log_line(" Short-response rows pruned: {$shortPruned}");
|
|
||||||
|
|
||||||
// 3. Trim whitespace on all text columns for the auto-generated ones
|
|
||||||
JarvisDB::execute(
|
|
||||||
"UPDATE kb_intents SET
|
|
||||||
intent_name = TRIM(intent_name),
|
|
||||||
pattern = TRIM(pattern),
|
|
||||||
response_template = TRIM(response_template),
|
|
||||||
fact_category = TRIM(fact_category)
|
|
||||||
WHERE priority = 5"
|
|
||||||
);
|
|
||||||
log_line(' Whitespace trimmed on all generated intents.');
|
|
||||||
|
|
||||||
// 4. Deactivate intents whose pattern is invalid PCRE
|
|
||||||
$all = JarvisDB::query('SELECT id, pattern FROM kb_intents WHERE active=1 AND priority=5');
|
|
||||||
$badPattern = 0;
|
|
||||||
foreach ($all as $row) {
|
|
||||||
if (@preg_match($row['pattern'], '') === false) {
|
|
||||||
JarvisDB::execute('UPDATE kb_intents SET active=0 WHERE id=?', [$row['id']]);
|
|
||||||
$badPattern++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log_line(" Bad PCRE patterns deactivated: {$badPattern}");
|
|
||||||
|
|
||||||
// 5. Stats
|
|
||||||
$stats = JarvisDB::single('SELECT COUNT(*) AS total, SUM(active) AS active FROM kb_intents');
|
|
||||||
log_line("Final KB Intents table: {$stats['total']} total, {$stats['active']} active.");
|
|
||||||
|
|
||||||
/* ── record last-run timestamp in kb_facts ── */
|
|
||||||
JarvisDB::execute(
|
|
||||||
"INSERT INTO kb_facts (category, fact_key, fact_value, host)
|
|
||||||
VALUES ('kb_generator', 'last_run', NOW(), 'local')
|
|
||||||
ON DUPLICATE KEY UPDATE fact_value=NOW(), updated_at=NOW()",
|
|
||||||
[]
|
|
||||||
);
|
|
||||||
JarvisDB::execute(
|
|
||||||
"INSERT INTO kb_facts (category, fact_key, fact_value, host)
|
|
||||||
VALUES ('kb_generator', 'last_inserted', ?, 'local')
|
|
||||||
ON DUPLICATE KEY UPDATE fact_value=VALUES(fact_value), updated_at=NOW()",
|
|
||||||
[$inserted]
|
|
||||||
);
|
|
||||||
|
|
||||||
log_line('Done.');
|
|
||||||
@@ -1,270 +0,0 @@
|
|||||||
<?php
|
|
||||||
/**
|
|
||||||
* JARVIS KB Intent Generator
|
|
||||||
* Generates 1,000+ educational KB intents via Groq LLM and imports to kb_intents table.
|
|
||||||
* Also runs a cleanup pass: deduplication, short-response pruning, pattern normalisation.
|
|
||||||
*
|
|
||||||
* CLI / cron: /usr/bin/php8.3 /var/www/jarvis/api/endpoints/kb_intent_generator.php
|
|
||||||
* Schedule: Daily – 3 am
|
|
||||||
*/
|
|
||||||
|
|
||||||
require_once __DIR__ . '/../config.php';
|
|
||||||
require_once __DIR__ . '/../lib/db.php';
|
|
||||||
|
|
||||||
/* ── helpers ── */
|
|
||||||
function ts(): string { return '[' . date('Y-m-d H:i:s') . ']'; }
|
|
||||||
function log_line(string $msg): void { echo ts() . ' KB Intent Generator: ' . $msg . "\n"; flush(); }
|
|
||||||
|
|
||||||
function groq(string $system, string $user, int $max = 3000, int $retries = 2): ?string {
|
|
||||||
for ($attempt = 0; $attempt <= $retries; $attempt++) {
|
|
||||||
if ($attempt > 0) {
|
|
||||||
log_line(" Retry {$attempt}/{$retries} after rate-limit pause...");
|
|
||||||
sleep(25);
|
|
||||||
}
|
|
||||||
$ch = curl_init('https://api.groq.com/openai/v1/chat/completions');
|
|
||||||
curl_setopt_array($ch, [
|
|
||||||
CURLOPT_RETURNTRANSFER => true,
|
|
||||||
CURLOPT_POST => true,
|
|
||||||
CURLOPT_TIMEOUT => 60,
|
|
||||||
CURLOPT_HTTPHEADER => [
|
|
||||||
'Authorization: Bearer ' . GROQ_API_KEY,
|
|
||||||
'Content-Type: application/json',
|
|
||||||
],
|
|
||||||
CURLOPT_POSTFIELDS => json_encode([
|
|
||||||
'model' => 'llama-3.3-70b-versatile',
|
|
||||||
'max_tokens' => $max,
|
|
||||||
'temperature' => 0.7,
|
|
||||||
'messages' => [
|
|
||||||
['role' => 'system', 'content' => $system],
|
|
||||||
['role' => 'user', 'content' => $user],
|
|
||||||
],
|
|
||||||
]),
|
|
||||||
]);
|
|
||||||
$raw = curl_exec($ch);
|
|
||||||
$err = curl_error($ch);
|
|
||||||
$info = curl_getinfo($ch);
|
|
||||||
curl_close($ch);
|
|
||||||
|
|
||||||
if ($err || !$raw) continue;
|
|
||||||
|
|
||||||
// Check for rate limit response (429)
|
|
||||||
if (($info['http_code'] ?? 0) === 429) continue;
|
|
||||||
|
|
||||||
$d = json_decode($raw, true);
|
|
||||||
$content = $d['choices'][0]['message']['content'] ?? null;
|
|
||||||
if ($content !== null) return $content;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ── normalize a pattern to valid PHP PCRE ── */
|
|
||||||
function normalize_pattern(string $pat): string {
|
|
||||||
$pat = trim($pat);
|
|
||||||
// If pattern already has PCRE delimiters, leave it alone
|
|
||||||
if (preg_match('/^[\/|~#!@%]/', $pat)) return $pat;
|
|
||||||
// Strip any (?i) inline flag — we'll add /i at the delimiter level
|
|
||||||
$pat = preg_replace('/^\(\?i\)/', '', $pat);
|
|
||||||
// Escape forward slashes inside the pattern
|
|
||||||
$pat = str_replace('/', '\\/', $pat);
|
|
||||||
return '/' . $pat . '/i';
|
|
||||||
}
|
|
||||||
|
|
||||||
/* ── run guard: skip if ran within last 4 hours ── */
|
|
||||||
/* Set JARVIS_FORCE_RUN=1 (env) or pass --force (argv) to bypass */
|
|
||||||
/* Comparison done in SQL (NOW()) rather than PHP time()/strtotime() — this process's
|
|
||||||
date_default_timezone_set('America/Chicago') makes strtotime() misread MySQL's naive
|
|
||||||
(UTC) timestamps as Chicago time, throwing elapsed-time checks off by the UTC offset. */
|
|
||||||
$recentRun = JarvisDB::single(
|
|
||||||
"SELECT (updated_at > DATE_SUB(NOW(), INTERVAL 14400 SECOND)) AS is_recent FROM kb_facts WHERE category='kb_generator' AND fact_key='last_run'"
|
|
||||||
);
|
|
||||||
$forceRun = !empty(getenv('JARVIS_FORCE_RUN')) || (isset($argv[1]) && $argv[1] === '--force');
|
|
||||||
if (!$forceRun && $recentRun && $recentRun['is_recent']) {
|
|
||||||
log_line('Skipping – ran within last 4 hours. Use --force to override.');
|
|
||||||
exit(0);
|
|
||||||
}
|
|
||||||
if ($forceRun) log_line('Force-run flag set — bypassing 4-hour guard.');
|
|
||||||
|
|
||||||
log_line('Starting daily KB intent generation run.');
|
|
||||||
|
|
||||||
/* ── load active topics from database ── */
|
|
||||||
$BATCHES = JarvisDB::query(
|
|
||||||
"SELECT t.topic_id AS id, t.category, t.topic_name AS topic, t.description AS `desc`
|
|
||||||
FROM kb_generator_topics t WHERE t.active=1 ORDER BY t.id ASC"
|
|
||||||
);
|
|
||||||
if (empty($BATCHES)) {
|
|
||||||
log_line('ERROR: No active topics in kb_generator_topics table. Add topics via the JARVIS admin panel.');
|
|
||||||
exit(1);
|
|
||||||
}
|
|
||||||
log_line('Loaded ' . count($BATCHES) . ' active topics from database.');
|
|
||||||
|
|
||||||
/* ── ROTATION ENGINE: process BATCH_SIZE topics per run, cycling through all ── */
|
|
||||||
define('BATCH_SIZE', 25);
|
|
||||||
$totalTopics = count($BATCHES);
|
|
||||||
$offsetRow = JarvisDB::single(
|
|
||||||
"SELECT CAST(fact_value AS SIGNED) AS v FROM kb_facts WHERE category='kb_generator' AND fact_key='batch_offset'"
|
|
||||||
);
|
|
||||||
$batchOffset = max(0, (int)($offsetRow['v'] ?? 0));
|
|
||||||
if ($batchOffset >= $totalTopics) $batchOffset = 0;
|
|
||||||
$nextOffset = ($batchOffset + BATCH_SIZE) % $totalTopics;
|
|
||||||
$cycleComplete = ($batchOffset + BATCH_SIZE) >= $totalTopics;
|
|
||||||
$cycleLen = (int)ceil($totalTopics / BATCH_SIZE);
|
|
||||||
|
|
||||||
$runBatches = [];
|
|
||||||
for ($i = 0; $i < BATCH_SIZE; $i++) {
|
|
||||||
$runBatches[] = $BATCHES[($batchOffset + $i) % $totalTopics];
|
|
||||||
}
|
|
||||||
$endIdx = ($batchOffset + BATCH_SIZE - 1) % $totalTopics;
|
|
||||||
log_line("Rotation: topics " . ($batchOffset + 1) . "–" . ($endIdx + 1) . " of {$totalTopics} | cycle = {$cycleLen} runs × 6h = " . ($cycleLen * 6) . "h full cycle.");
|
|
||||||
if ($cycleComplete) log_line(" ↻ Full cycle complete — restarting from topic 1 next run.");
|
|
||||||
|
|
||||||
/* ── main generation loop ── */
|
|
||||||
$totalBatches = count($runBatches);
|
|
||||||
foreach ($runBatches as $idx => $batch) {
|
|
||||||
$num = $idx + 1;
|
|
||||||
log_line("Batch {$num}/{$totalBatches}: {$batch['topic']}");
|
|
||||||
|
|
||||||
$user = "Generate 20 KB intents for the topic: {$batch['topic']}.\n"
|
|
||||||
. "Subtopics to cover: {$batch['desc']}.\n"
|
|
||||||
. "Prefix every intent_name with \"{$batch['id']}_\".\n"
|
|
||||||
. "Category string to use: \"{$batch['category']}\".";
|
|
||||||
|
|
||||||
$raw = groq($SYSTEM, $user, 5000);
|
|
||||||
if ($raw === null) {
|
|
||||||
log_line(" ✗ API call failed after retries – skipping batch.");
|
|
||||||
$errors += 20;
|
|
||||||
sleep(8);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strip markdown code fences if model added them
|
|
||||||
$raw = preg_replace('/^```(?:json)?\s*/m', '', $raw);
|
|
||||||
$raw = preg_replace('/^```\s*/m', '', $raw);
|
|
||||||
$raw = trim($raw);
|
|
||||||
|
|
||||||
// Extract JSON array; fall back to partial recovery for truncated responses
|
|
||||||
$items = null;
|
|
||||||
if (preg_match('/\[\s*\{.*\}\s*\]/s', $raw, $m)) {
|
|
||||||
$items = json_decode($m[0], true);
|
|
||||||
if (!is_array($items)) {
|
|
||||||
log_line(" ✗ JSON parse failed – skipping batch.");
|
|
||||||
$errors += 20; sleep(8); continue;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$start = strpos($raw, '[');
|
|
||||||
if ($start !== false) {
|
|
||||||
$partial = substr($raw, $start);
|
|
||||||
if (preg_match_all('/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/s', $partial, $objs) && !empty($objs[0])) {
|
|
||||||
$recovered = '[' . implode(',', $objs[0]) . ']';
|
|
||||||
$items = json_decode($recovered, true);
|
|
||||||
if (is_array($items) && count($items) > 0)
|
|
||||||
log_line(" ⚠ Truncated — recovered " . count($items) . " items.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!is_array($items) || count($items) === 0) {
|
|
||||||
log_line(" ✗ No JSON array found — raw[0:120]: " . substr(str_replace("\n", ' ', $raw), 0, 120));
|
|
||||||
$errors += 20; sleep(8); continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$batchInserted = 0;
|
|
||||||
foreach ($items as $item) {
|
|
||||||
if (!is_array($item)) continue;
|
|
||||||
safe_insert($item, $batch['category']);
|
|
||||||
$batchInserted++;
|
|
||||||
}
|
|
||||||
log_line(" ✓ Parsed {$batchInserted} intents (running total inserted: {$inserted}).");
|
|
||||||
|
|
||||||
// Polite delay between API calls — Groq TPM limit needs ~8s between batches
|
|
||||||
if ($num < $totalBatches) sleep(8);
|
|
||||||
}
|
|
||||||
|
|
||||||
log_line("Generation complete. Inserted/updated: {$inserted} | Short/invalid skipped: {$skipped} | Errors: {$errors}");
|
|
||||||
|
|
||||||
/* ── cleanup phase ── */
|
|
||||||
log_line('Starting cleanup phase...');
|
|
||||||
|
|
||||||
// 1. Remove exact duplicate intent_names (keep the one with the longer response)
|
|
||||||
$dups = JarvisDB::query(
|
|
||||||
'SELECT intent_name, COUNT(*) AS cnt FROM kb_intents GROUP BY intent_name HAVING cnt > 1'
|
|
||||||
);
|
|
||||||
$dupsPruned = 0;
|
|
||||||
foreach ($dups as $dup) {
|
|
||||||
$rows = JarvisDB::query(
|
|
||||||
'SELECT id, LENGTH(response_template) AS rlen FROM kb_intents WHERE intent_name=? ORDER BY rlen DESC',
|
|
||||||
[$dup['intent_name']]
|
|
||||||
);
|
|
||||||
array_shift($rows);
|
|
||||||
foreach ($rows as $row) {
|
|
||||||
JarvisDB::execute('DELETE FROM kb_intents WHERE id=?', [$row['id']]);
|
|
||||||
$dupsPruned++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log_line(" Duplicate intent_names pruned: {$dupsPruned}");
|
|
||||||
|
|
||||||
// 2. Remove intents with very short responses (< 40 chars)
|
|
||||||
$shortPruned = JarvisDB::execute(
|
|
||||||
"DELETE FROM kb_intents WHERE LENGTH(response_template) < 40 AND priority <= 5"
|
|
||||||
);
|
|
||||||
log_line(" Short-response rows pruned: {$shortPruned}");
|
|
||||||
|
|
||||||
// 3. Trim whitespace on all generated intents
|
|
||||||
JarvisDB::execute(
|
|
||||||
"UPDATE kb_intents SET
|
|
||||||
intent_name = TRIM(intent_name),
|
|
||||||
pattern = TRIM(pattern),
|
|
||||||
response_template = TRIM(response_template),
|
|
||||||
fact_category = TRIM(fact_category)
|
|
||||||
WHERE priority = 5"
|
|
||||||
);
|
|
||||||
log_line(' Whitespace trimmed on all generated intents.');
|
|
||||||
|
|
||||||
// 4. Fix and validate PCRE patterns — normalize then deactivate only truly broken ones
|
|
||||||
$all = JarvisDB::query('SELECT id, pattern FROM kb_intents WHERE priority=5');
|
|
||||||
$badPattern = 0;
|
|
||||||
$fixedPattern = 0;
|
|
||||||
foreach ($all as $row) {
|
|
||||||
$pat = normalize_pattern($row['pattern']);
|
|
||||||
if ($pat !== $row['pattern']) {
|
|
||||||
// Update to normalized form
|
|
||||||
JarvisDB::execute('UPDATE kb_intents SET pattern=?, active=1 WHERE id=?', [$pat, $row['id']]);
|
|
||||||
$fixedPattern++;
|
|
||||||
} elseif (@preg_match($pat, '') === false) {
|
|
||||||
JarvisDB::execute('UPDATE kb_intents SET active=0 WHERE id=?', [$row['id']]);
|
|
||||||
$badPattern++;
|
|
||||||
} else {
|
|
||||||
// Valid pattern — make sure it's active
|
|
||||||
JarvisDB::execute('UPDATE kb_intents SET active=1 WHERE id=?', [$row['id']]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log_line(" Patterns normalized: {$fixedPattern} | Bad PCRE deactivated: {$badPattern}");
|
|
||||||
|
|
||||||
// 5. Enable ALL remaining inactive intents (including pre-existing ones)
|
|
||||||
$reactivated = JarvisDB::execute('UPDATE kb_intents SET active=1 WHERE active=0 AND priority <= 5');
|
|
||||||
log_line(" Re-activated previously inactive intents: {$reactivated}");
|
|
||||||
|
|
||||||
// 6. Final stats
|
|
||||||
$stats = JarvisDB::single('SELECT COUNT(*) AS total, SUM(active) AS active FROM kb_intents');
|
|
||||||
log_line("Final KB Intents table: {$stats['total']} total, {$stats['active']} active.");
|
|
||||||
|
|
||||||
/* ── record last-run timestamp ── */
|
|
||||||
JarvisDB::execute(
|
|
||||||
"INSERT INTO kb_facts (category, fact_key, fact_value, host)
|
|
||||||
VALUES ('kb_generator', 'last_run', NOW(), 'local')
|
|
||||||
ON DUPLICATE KEY UPDATE fact_value=NOW(), updated_at=NOW()",
|
|
||||||
[]
|
|
||||||
);
|
|
||||||
JarvisDB::execute(
|
|
||||||
"INSERT INTO kb_facts (category, fact_key, fact_value, host)
|
|
||||||
VALUES ('kb_generator', 'batch_offset', ?, 'local')
|
|
||||||
ON DUPLICATE KEY UPDATE fact_value=VALUES(fact_value), updated_at=NOW()",
|
|
||||||
[$nextOffset]
|
|
||||||
);
|
|
||||||
log_line("Next run will start at topic offset {$nextOffset}/{$totalTopics}.");
|
|
||||||
JarvisDB::execute(
|
|
||||||
"INSERT INTO kb_facts (category, fact_key, fact_value, host)
|
|
||||||
VALUES ('kb_generator', 'last_inserted', ?, 'local')
|
|
||||||
ON DUPLICATE KEY UPDATE fact_value=VALUES(fact_value), updated_at=NOW()",
|
|
||||||
[$inserted]
|
|
||||||
);
|
|
||||||
|
|
||||||
log_line('Done.');
|
|
||||||
Reference in New Issue
Block a user