jarvis/api/endpoints/kb_intent_generator.php

<?php
/**
 * JARVIS KB Intent Generator
 * Generates 1,000+ educational KB intents via Groq LLM and imports to kb_intents table.
 * Also runs a cleanup pass: deduplication, short-response pruning, pattern normalisation.
 *
 * CLI / cron:  /usr/bin/php8.3 /var/www/jarvis/api/endpoints/kb_intent_generator.php
 * Schedule:    Daily – 3 am
 */

require_once __DIR__ . '/../config.php';
require_once __DIR__ . '/../lib/db.php';

/* ── helpers ── */
function ts(): string { return '[' . date('Y-m-d H:i:s') . ']'; }
function log_line(string $msg): void { echo ts() . ' KB Intent Generator: ' . $msg . "\n"; flush(); }

function groq(string $system, string $user, int $max = 3000, int $retries = 2): ?string {
    for ($attempt = 0; $attempt <= $retries; $attempt++) {
        if ($attempt > 0) {
            log_line("  Retry {$attempt}/{$retries} after rate-limit pause...");
            sleep(25);
        }
        $ch = curl_init('https://api.groq.com/openai/v1/chat/completions');
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_POST           => true,
            CURLOPT_TIMEOUT        => 60,
            CURLOPT_HTTPHEADER     => [
                'Authorization: Bearer ' . GROQ_API_KEY,
                'Content-Type: application/json',
            ],
            CURLOPT_POSTFIELDS => json_encode([
                'model'       => 'llama-3.3-70b-versatile',
                'max_tokens'  => $max,
                'temperature' => 0.7,
                'messages'    => [
                    ['role' => 'system', 'content' => $system],
                    ['role' => 'user',   'content' => $user],
                ],
            ]),
        ]);
        $raw  = curl_exec($ch);
        $err  = curl_error($ch);
        $info = curl_getinfo($ch);
        curl_close($ch);

        if ($err || !$raw) continue;

        // Check for rate limit response (429)
        if (($info['http_code'] ?? 0) === 429) continue;

        $d = json_decode($raw, true);
        $content = $d['choices'][0]['message']['content'] ?? null;
        if ($content !== null) return $content;
    }
    return null;
}

/* ── normalize a pattern to valid PHP PCRE ── */
function normalize_pattern(string $pat): string {
    $pat = trim($pat);
    // If pattern already has PCRE delimiters, leave it alone
    if (preg_match('/^[\/|~#!@%]/', $pat)) return $pat;
    // Strip any (?i) inline flag — we'll add /i at the delimiter level
    $pat = preg_replace('/^\(\?i\)/', '', $pat);
    // Escape forward slashes inside the pattern
    $pat = str_replace('/', '\\/', $pat);
    return '/' . $pat . '/i';
}

/* ── run guard: skip if ran within last 4 hours ── */
/* Set JARVIS_FORCE_RUN=1 (env) or pass --force (argv) to bypass */
/* Comparison done in SQL (NOW()) rather than PHP time()/strtotime() — this process's
   date_default_timezone_set('America/Chicago') makes strtotime() misread MySQL's naive
   (UTC) timestamps as Chicago time, throwing elapsed-time checks off by the UTC offset. */
$recentRun = JarvisDB::single(
    "SELECT (updated_at > DATE_SUB(NOW(), INTERVAL 14400 SECOND)) AS is_recent FROM kb_facts WHERE category='kb_generator' AND fact_key='last_run'"
);
$forceRun = !empty(getenv('JARVIS_FORCE_RUN')) || (isset($argv[1]) && $argv[1] === '--force');
if (!$forceRun && $recentRun && $recentRun['is_recent']) {
    log_line('Skipping – ran within last 4 hours. Use --force to override.');
    exit(0);
}
if ($forceRun) log_line('Force-run flag set — bypassing 4-hour guard.');

log_line('Starting daily KB intent generation run.');

/* ── load active topics from database ── */
$BATCHES = JarvisDB::query(
    "SELECT t.topic_id AS id, t.category, t.topic_name AS topic, t.description AS `desc`
     FROM kb_generator_topics t WHERE t.active=1 ORDER BY t.id ASC"
);
if (empty($BATCHES)) {
    log_line('ERROR: No active topics in kb_generator_topics table. Add topics via the JARVIS admin panel.');
    exit(1);
}
log_line('Loaded ' . count($BATCHES) . ' active topics from database.');

/* ── ROTATION ENGINE: process BATCH_SIZE topics per run, cycling through all ── */
define('BATCH_SIZE', 25);
$totalTopics = count($BATCHES);
$offsetRow   = JarvisDB::single(
    "SELECT CAST(fact_value AS SIGNED) AS v FROM kb_facts WHERE category='kb_generator' AND fact_key='batch_offset'"
);
$batchOffset = max(0, (int)($offsetRow['v'] ?? 0));
if ($batchOffset >= $totalTopics) $batchOffset = 0;
$nextOffset    = ($batchOffset + BATCH_SIZE) % $totalTopics;
$cycleComplete = ($batchOffset + BATCH_SIZE) >= $totalTopics;
$cycleLen      = (int)ceil($totalTopics / BATCH_SIZE);

$runBatches = [];
for ($i = 0; $i < BATCH_SIZE; $i++) {
    $runBatches[] = $BATCHES[($batchOffset + $i) % $totalTopics];
}
$endIdx = ($batchOffset + BATCH_SIZE - 1) % $totalTopics;
log_line("Rotation: topics " . ($batchOffset + 1) . "–" . ($endIdx + 1) . " of {$totalTopics} | cycle = {$cycleLen} runs × 6h = " . ($cycleLen * 6) . "h full cycle.");
if ($cycleComplete) log_line("  ↻ Full cycle complete — restarting from topic 1 next run.");

/* ── system prompt ── */
/* RESTORED 2026-07-07: this and safe_insert() below were lost during the 2026-07-05 rotation-engine
   refactor (moving from a hardcoded $BATCHES array to the kb_generator_topics table). Their absence
   caused an uncaught TypeError on every run since (undefined $SYSTEM passed to groq()'s non-nullable
   string param), silently swallowed by config.php's error_reporting(0) — the cron looked like it was
   running fine but died instantly on batch 1 every single time. Restored from kb_intent_generator.php.bak2,
   with the intent count adjusted from 40 to 20 to match this version's actual per-topic request below. */
$SYSTEM = <<<'SYS'
You are an expert educator generating KB (knowledge-base) intents for an AI assistant called JARVIS.
Each intent is a question/phrase a student might ask, paired with a clear educational answer.

Respond ONLY with a valid JSON array (no markdown, no backticks, no commentary).
Each element must have exactly these keys:
  "n"  – intent_name: unique snake_case identifier ≤ 60 chars, prefixed with the batch id given
  "p"  – pattern: a PHP PCRE regex (use (?i) for case-insensitive) that matches the question
  "r"  – response: a thorough but concise educational answer (2–5 sentences or a short structured list)
  "c"  – category: the category string provided

Rules:
- Patterns must use \\b word boundaries; escape backslashes for JSON (\\b not \b)
- Patterns should NOT start with ^ or end with $ (they are substring matches)
- Responses must be factually accurate
- Do not duplicate intent names; every "n" must be unique within this batch
- Return exactly 20 intents
SYS;

/* ── insert helper ── */
$inserted = 0;
$skipped  = 0;
$errors   = 0;

function safe_insert(array $intent, string $batchCategory): void {
    global $inserted, $skipped, $errors;
    $name     = trim($intent['n'] ?? '');
    $pattern  = trim($intent['p'] ?? '');
    $response = trim($intent['r'] ?? '');
    $category = trim($intent['c'] ?? $batchCategory);

    if (!$name || !$pattern || !$response) { $errors++; return; }
    if (strlen($name) > 64)     $name     = substr($name, 0, 64);
    if (strlen($pattern) > 512) $pattern  = substr($pattern, 0, 512);
    if (strlen($response) < 30) { $skipped++; return; }   // too short

    try {
        JarvisDB::execute(
            'INSERT INTO kb_intents (intent_name, pattern, response_template, fact_category, action_type, priority, active)
             VALUES (?, ?, ?, ?, ?, ?, ?)
             ON DUPLICATE KEY UPDATE
               pattern=VALUES(pattern),
               response_template=VALUES(response_template),
               fact_category=VALUES(fact_category)',
            [$name, $pattern, $response, $category, 'response', 5, 1]
        );
        $inserted++;
    } catch (Exception $e) {
        $errors++;
    }
}

/* ── main generation loop ── */
$totalBatches = count($runBatches);
foreach ($runBatches as $idx => $batch) {
    $num = $idx + 1;
    log_line("Batch {$num}/{$totalBatches}: {$batch['topic']}");

    $user = "Generate 20 KB intents for the topic: {$batch['topic']}.\n"
          . "Subtopics to cover: {$batch['desc']}.\n"
          . "Prefix every intent_name with \"{$batch['id']}_\".\n"
          . "Category string to use: \"{$batch['category']}\".";

    $raw = groq($SYSTEM, $user, 5000);
    if ($raw === null) {
        log_line("  ✗ API call failed after retries – skipping batch.");
        $errors += 20;
        sleep(8);
        continue;
    }

    // Strip markdown code fences if model added them
    $raw = preg_replace('/^```(?:json)?\s*/m', '', $raw);
    $raw = preg_replace('/^```\s*/m', '', $raw);
    $raw = trim($raw);

    // Extract JSON array; fall back to partial recovery for truncated responses
    $items = null;
    if (preg_match('/\[\s*\{.*\}\s*\]/s', $raw, $m)) {
        $items = json_decode($m[0], true);
        if (!is_array($items)) {
            log_line("  ✗ JSON parse failed – skipping batch.");
            $errors += 20; sleep(8); continue;
        }
    } else {
        $start = strpos($raw, '[');
        if ($start !== false) {
            $partial = substr($raw, $start);
            if (preg_match_all('/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/s', $partial, $objs) && !empty($objs[0])) {
                $recovered = '[' . implode(',', $objs[0]) . ']';
                $items = json_decode($recovered, true);
                if (is_array($items) && count($items) > 0)
                    log_line("  ⚠ Truncated — recovered " . count($items) . " items.");
            }
        }
        if (!is_array($items) || count($items) === 0) {
            log_line("  ✗ No JSON array found — raw[0:120]: " . substr(str_replace("\n", ' ', $raw), 0, 120));
            $errors += 20; sleep(8); continue;
        }
    }

    $batchInserted = 0;
    foreach ($items as $item) {
        if (!is_array($item)) continue;
        safe_insert($item, $batch['category']);
        $batchInserted++;
    }
    log_line("  ✓ Parsed {$batchInserted} intents (running total inserted: {$inserted}).");

    // Polite delay between API calls — Groq TPM limit needs ~8s between batches
    if ($num < $totalBatches) sleep(8);
}

log_line("Generation complete. Inserted/updated: {$inserted} | Short/invalid skipped: {$skipped} | Errors: {$errors}");

/* ── cleanup phase ── */
log_line('Starting cleanup phase...');

// 1. Remove exact duplicate intent_names (keep the one with the longer response)
$dups = JarvisDB::query(
    'SELECT intent_name, COUNT(*) AS cnt FROM kb_intents GROUP BY intent_name HAVING cnt > 1'
);
$dupsPruned = 0;
foreach ($dups as $dup) {
    $rows = JarvisDB::query(
        'SELECT id, LENGTH(response_template) AS rlen FROM kb_intents WHERE intent_name=? ORDER BY rlen DESC',
        [$dup['intent_name']]
    );
    array_shift($rows);
    foreach ($rows as $row) {
        JarvisDB::execute('DELETE FROM kb_intents WHERE id=?', [$row['id']]);
        $dupsPruned++;
    }
}
log_line("  Duplicate intent_names pruned: {$dupsPruned}");

// 2. Remove intents with very short responses (< 40 chars)
$shortPruned = JarvisDB::execute(
    "DELETE FROM kb_intents WHERE LENGTH(response_template) < 40 AND priority <= 5"
);
log_line("  Short-response rows pruned: {$shortPruned}");

// 3. Trim whitespace on all generated intents
JarvisDB::execute(
    "UPDATE kb_intents SET
        intent_name       = TRIM(intent_name),
        pattern           = TRIM(pattern),
        response_template = TRIM(response_template),
        fact_category     = TRIM(fact_category)
     WHERE priority = 5"
);
log_line('  Whitespace trimmed on all generated intents.');

// 4. Fix and validate PCRE patterns — normalize then deactivate only truly broken ones
$all = JarvisDB::query('SELECT id, pattern FROM kb_intents WHERE priority=5');
$badPattern  = 0;
$fixedPattern = 0;
foreach ($all as $row) {
    $pat = normalize_pattern($row['pattern']);
    if ($pat !== $row['pattern']) {
        // Update to normalized form
        JarvisDB::execute('UPDATE kb_intents SET pattern=?, active=1 WHERE id=?', [$pat, $row['id']]);
        $fixedPattern++;
    } elseif (@preg_match($pat, '') === false) {
        JarvisDB::execute('UPDATE kb_intents SET active=0 WHERE id=?', [$row['id']]);
        $badPattern++;
    } else {
        // Valid pattern — make sure it's active
        JarvisDB::execute('UPDATE kb_intents SET active=1 WHERE id=?', [$row['id']]);
    }
}
log_line("  Patterns normalized: {$fixedPattern} | Bad PCRE deactivated: {$badPattern}");

// 5. Enable ALL remaining inactive intents (including pre-existing ones)
$reactivated = JarvisDB::execute('UPDATE kb_intents SET active=1 WHERE active=0 AND priority <= 5');
log_line("  Re-activated previously inactive intents: {$reactivated}");

// 6. Final stats
$stats = JarvisDB::single('SELECT COUNT(*) AS total, SUM(active) AS active FROM kb_intents');
log_line("Final KB Intents table: {$stats['total']} total, {$stats['active']} active.");

/* ── record last-run timestamp ── */
JarvisDB::execute(
    "INSERT INTO kb_facts (category, fact_key, fact_value, host)
     VALUES ('kb_generator', 'last_run', NOW(), 'local')
     ON DUPLICATE KEY UPDATE fact_value=NOW(), updated_at=NOW()",
    []
);
JarvisDB::execute(
    "INSERT INTO kb_facts (category, fact_key, fact_value, host)
     VALUES ('kb_generator', 'batch_offset', ?, 'local')
     ON DUPLICATE KEY UPDATE fact_value=VALUES(fact_value), updated_at=NOW()",
    [$nextOffset]
);
log_line("Next run will start at topic offset {$nextOffset}/{$totalTopics}.");
JarvisDB::execute(
    "INSERT INTO kb_facts (category, fact_key, fact_value, host)
     VALUES ('kb_generator', 'last_inserted', ?, 'local')
     ON DUPLICATE KEY UPDATE fact_value=VALUES(fact_value), updated_at=NOW()",
    [$inserted]
);

log_line('Done.');