token-monitor/analyze.js

#!/usr/bin/env node
/**
 * analyze.js — Token Monitor analysis CLI
 *
 * Reads accumulated JSONL logs from ~/.logs/token-monitor/ and produces:
 *   - Burn rates per account (utilization delta over time)
 *   - Weekly budget reconstruction
 *   - Cycle stagger view (next 48h resets)
 *   - Rotation recommendations (rule-based)
 *   - Underspend alerts
 *   - Log hygiene (--prune)
 *
 * Usage:
 *   node analyze.js                         # full report
 *   node analyze.js --burn-rate             # burn rate section only
 *   node analyze.js --weekly                # weekly reconstruction only
 *   node analyze.js --stagger               # cycle stagger only
 *   node analyze.js --rotation              # rotation recommendation only
 *   node analyze.js --json                  # JSON output (all sections)
 *   node analyze.js --provider team-nadja   # filter to one provider
 *   node analyze.js --prune [--dry-run]     # log hygiene
 */

import {
  readdirSync, readFileSync, writeFileSync, mkdirSync, unlinkSync, existsSync,
} from 'fs';
import { homedir } from 'os';
import { join } from 'path';

const LOG_DIR = join(homedir(), '.logs', 'token-monitor');
const TEAMS = ['team-vigilio', 'team-ludo', 'team-molto', 'team-nadja', 'team-buio'];

// ── Load logs ────────────────────────────────────────────────────────────────

function loadLogs(providerFilter = null) {
  if (!existsSync(LOG_DIR)) return [];

  const files = readdirSync(LOG_DIR)
    .filter(f => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f))
    .sort();

  const entries = [];
  for (const file of files) {
    const content = readFileSync(join(LOG_DIR, file), 'utf-8').trim();
    if (!content) continue;
    for (const line of content.split('\n').filter(Boolean)) {
      try {
        const entry = JSON.parse(line);
        const providers = entry.providers || {};
        // Skip test/empty entries — real entries have at least one provider with a type
        if (!Object.values(providers).some(p => p && p.type)) continue;
        if (providerFilter && !providers[providerFilter]) continue;
        entries.push(entry);
      } catch { /* skip bad lines */ }
    }
  }

  return entries.sort((a, b) => a.ts.localeCompare(b.ts));
}

// ── Helpers ───────────────────────────────────────────────────────────────────

function formatDuration(seconds) {
  if (seconds == null || isNaN(seconds) || seconds < 0) return '?';
  const h = Math.floor(seconds / 3600);
  const m = Math.floor((seconds % 3600) / 60);
  if (h > 0) return `${h}h ${m > 0 ? m + 'm' : ''}`.trim();
  if (m > 0) return `${m}m`;
  return `${Math.round(seconds)}s`;
}

function pct(v) {
  if (v == null) return '?';
  return `${Math.round(v * 100)}%`;
}

function getISOWeek(dateStr) {
  const d = new Date(dateStr);
  d.setUTCHours(12, 0, 0, 0);
  d.setUTCDate(d.getUTCDate() + 3 - (d.getUTCDay() + 6) % 7);
  const week1 = new Date(Date.UTC(d.getUTCFullYear(), 0, 4));
  const weekNum = 1 + Math.round(((d - week1) / 86400000 - 3 + (week1.getUTCDay() + 6) % 7) / 7);
  return `${d.getUTCFullYear()}-W${String(weekNum).padStart(2, '0')}`;
}

function latestPerProvider(entries, typeFilter = null) {
  const latest = {};
  for (const entry of entries) {
    for (const [name, p] of Object.entries(entry.providers || {})) {
      if (typeFilter && p?.type !== typeFilter) continue;
      if (!latest[name] || entry.ts > latest[name].ts) {
        latest[name] = { ts: entry.ts, p };
      }
    }
  }
  return latest;
}

// ── Burn rate ─────────────────────────────────────────────────────────────────

function computeBurnRate(entries, providerName) {
  const pts = entries
    .filter(e => e.providers[providerName]?.utilization_7d != null)
    .map(e => ({
      ts: new Date(e.ts).getTime(),
      util7d: e.providers[providerName].utilization_7d,
    }));

  if (pts.length < 2) return null;

  const first = pts[0];
  const last = pts[pts.length - 1];
  const hours = (last.ts - first.ts) / 3_600_000;
  if (hours < 0.01) return null;

  const rate = (last.util7d - first.util7d) / hours;
  const exhaustion = rate > 0 ? (1 - last.util7d) / rate : null;

  return {
    rate_per_hour: rate,
    projected_exhaustion_hours: exhaustion,
    current_util_7d: last.util7d,
    first_util_7d: first.util7d,
    data_points: pts.length,
    hours_elapsed: hours,
    first_ts: new Date(first.ts).toISOString(),
    last_ts: new Date(last.ts).toISOString(),
  };
}

// ── Weekly reconstruction ─────────────────────────────────────────────────────

function reconstructWeekly(entries) {
  const weeks = {};

  for (const entry of entries) {
    const week = getISOWeek(entry.ts);
    if (!weeks[week]) weeks[week] = { providers: {} };

    const w = weeks[week];
    const dateStr = entry.ts.slice(0, 10);
    if (!w.start || dateStr < w.start) w.start = dateStr;
    if (!w.end || dateStr > w.end) w.end = dateStr;

    for (const [name, p] of Object.entries(entry.providers || {})) {
      if (p?.type !== 'teams-direct') continue;
      if (!w.providers[name]) {
        w.providers[name] = {
          samples: 0, peak_util_5h: 0, peak_util_7d: 0,
          _total_util_7d: 0, exhausted_count: 0,
        };
      }
      const s = w.providers[name];
      s.samples++;
      if (p.utilization_5h != null) s.peak_util_5h = Math.max(s.peak_util_5h, p.utilization_5h);
      if (p.utilization_7d != null) {
        s.peak_util_7d = Math.max(s.peak_util_7d, p.utilization_7d);
        s._total_util_7d += p.utilization_7d;
      }
      if (p.status === 'rejected') s.exhausted_count++;
    }
  }

  // Finalize averages, remove internal accumulator
  for (const w of Object.values(weeks)) {
    for (const s of Object.values(w.providers)) {
      s.avg_util_7d = s.samples > 0 ? s._total_util_7d / s.samples : 0;
      delete s._total_util_7d;
    }
  }

  return weeks;
}

// ── Cycle stagger ─────────────────────────────────────────────────────────────

function cycleStagger(entries) {
  const latest = latestPerProvider(entries, 'teams-direct');
  const now = Date.now();
  const results = [];

  for (const [provider, { ts, p }] of Object.entries(latest)) {
    if (p.reset_in_seconds == null) continue;
    const entryAgeSeconds = (now - new Date(ts).getTime()) / 1000;
    const resetInSecondsNow = Math.max(0, p.reset_in_seconds - entryAgeSeconds);
    if (resetInSecondsNow > 172_800) continue; // > 48h, skip

    results.push({
      provider,
      resets_at_iso: new Date(now + resetInSecondsNow * 1000).toISOString(),
      resets_in_seconds_from_now: Math.round(resetInSecondsNow),
    });
  }

  return results.sort((a, b) => a.resets_in_seconds_from_now - b.resets_in_seconds_from_now);
}

// ── Underspend alerts ─────────────────────────────────────────────────────────

function underspendAlerts(entries) {
  const latest = latestPerProvider(entries, 'teams-direct');
  const alerts = [];

  for (const [provider, { p }] of Object.entries(latest)) {
    if (p.utilization_5h == null) continue;
    if (p.status !== 'allowed') continue;
    if (p.utilization_5h < 0.60 && p.reset_in_seconds != null && p.reset_in_seconds < 7200) {
      alerts.push({ provider, utilization_5h: p.utilization_5h, reset_in_seconds: p.reset_in_seconds });
    }
  }

  return alerts;
}

// ── Rotation rank ─────────────────────────────────────────────────────────────

function rotationRank(entries) {
  const latest = latestPerProvider(entries, 'teams-direct');
  const ranked = [];

  for (const name of TEAMS) {
    const rec = latest[name];
    if (!rec) continue;
    const p = rec.p;

    let score, reason, severity;

    if (p.status === 'invalid_key') {
      score = -200;
      reason = '401 invalid key — cannot use';
      severity = 'unknown';
    } else if (p.status === 'rejected') {
      const resetIn = p.reset_in_seconds || 999_999;
      // Among maxed accounts, soonest reset gets slight priority
      score = -100 + (1 / (resetIn + 1));
      reason = `MAXED — avoid until reset in ${formatDuration(resetIn)}`;
      severity = 'critical';
    } else if (p.utilization_7d == null && p.utilization_5h == null) {
      score = 50;
      reason = 'DORMANT — hold in reserve for cycle staggering';
      severity = 'dormant';
    } else {
      const headroom = 1 - (p.utilization_7d || 0);
      score = headroom * 100;
      if (score < 30) {
        reason = `low headroom — 7d: ${pct(p.utilization_7d)}, use cautiously`;
        severity = 'warning';
      } else {
        const resetStr = p.reset_in_seconds != null ? `, resets ${formatDuration(p.reset_in_seconds)}` : '';
        reason = `${pct(headroom)} headroom — 7d: ${pct(p.utilization_7d)}${resetStr}`;
        severity = 'ok';
      }
    }

    ranked.push({ provider: name, score, reason, severity });
  }

  return ranked.sort((a, b) => b.score - a.score);
}

// ── Log hygiene ───────────────────────────────────────────────────────────────

function pruneLogs(dryRun = false) {
  if (!existsSync(LOG_DIR)) {
    console.log('No log directory — nothing to prune.');
    return;
  }

  const files = readdirSync(LOG_DIR).filter(f => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f));
  const cutoff = new Date(Date.now() - 30 * 86_400_000).toISOString().slice(0, 10);
  const toPrune = files.filter(f => f.slice(0, 10) < cutoff);

  if (toPrune.length === 0) {
    console.log('No files older than 30 days — nothing to prune.');
    return;
  }

  const weeksDir = join(LOG_DIR, 'weeks');
  if (!dryRun) mkdirSync(weeksDir, { recursive: true });

  const weeklyAgg = {};

  for (const file of toPrune) {
    const dateStr = file.slice(0, 10);
    const week = getISOWeek(dateStr + 'T12:00:00Z');
    const content = readFileSync(join(LOG_DIR, file), 'utf-8').trim();

    if (!weeklyAgg[week]) {
      weeklyAgg[week] = { week, start: dateStr, end: dateStr, providers: {} };
    }
    const w = weeklyAgg[week];
    if (dateStr < w.start) w.start = dateStr;
    if (dateStr > w.end) w.end = dateStr;

    for (const line of content.split('\n').filter(Boolean)) {
      try {
        const entry = JSON.parse(line);
        for (const [name, p] of Object.entries(entry.providers || {})) {
          if (p?.type !== 'teams-direct') continue;
          if (!w.providers[name]) {
            w.providers[name] = {
              samples: 0, peak_util_5h: 0, peak_util_7d: 0, avg_util_7d: 0, exhausted_count: 0,
            };
          }
          const s = w.providers[name];
          if (p.utilization_5h != null) s.peak_util_5h = Math.max(s.peak_util_5h, p.utilization_5h);
          if (p.utilization_7d != null) {
            s.peak_util_7d = Math.max(s.peak_util_7d, p.utilization_7d);
            s.avg_util_7d = (s.avg_util_7d * s.samples + p.utilization_7d) / (s.samples + 1);
          }
          s.samples++;
          if (p.status === 'rejected') s.exhausted_count++;
        }
      } catch { /* skip */ }
    }
  }

  let pruned = 0;
  for (const [week, data] of Object.entries(weeklyAgg)) {
    const weekFile = join(weeksDir, `${week}.json`);
    if (dryRun) {
      console.log(`[dry-run] Would write ${weekFile}`);
    } else {
      if (existsSync(weekFile)) {
        // Merge with existing weekly file
        const existing = JSON.parse(readFileSync(weekFile, 'utf-8'));
        for (const [name, s] of Object.entries(data.providers)) {
          if (!existing.providers[name]) { existing.providers[name] = s; continue; }
          const e = existing.providers[name];
          const totalSamples = e.samples + s.samples;
          e.peak_util_5h = Math.max(e.peak_util_5h, s.peak_util_5h);
          e.peak_util_7d = Math.max(e.peak_util_7d, s.peak_util_7d);
          e.avg_util_7d = (e.avg_util_7d * e.samples + s.avg_util_7d * s.samples) / totalSamples;
          e.samples = totalSamples;
          e.exhausted_count += s.exhausted_count;
        }
        writeFileSync(weekFile, JSON.stringify(existing, null, 2));
      } else {
        writeFileSync(weekFile, JSON.stringify(data, null, 2));
      }
    }
  }

  for (const file of toPrune) {
    if (dryRun) {
      console.log(`[dry-run] Would delete ${join(LOG_DIR, file)}`);
    } else {
      unlinkSync(join(LOG_DIR, file));
      pruned++;
    }
  }

  if (dryRun) {
    console.log(`[dry-run] Would prune ${toPrune.length} file(s) into ${Object.keys(weeklyAgg).length} weekly summary file(s).`);
  } else {
    console.log(`Pruned ${pruned} file(s) into ${Object.keys(weeklyAgg).length} weekly summary file(s).`);
  }
}

// ── Report generation ─────────────────────────────────────────────────────────

function generateFullReport(entries) {
  const ts = new Date().toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' UTC');
  const width = 56;
  const lines = [
    `Token Analysis — ${ts}`,
    '═'.repeat(width),
    '',
  ];

  // ── Burn rates
  lines.push('Burn Rate');
  const latestTeams = latestPerProvider(entries, 'teams-direct');
  let anyTeams = false;
  for (const name of TEAMS) {
    const rec = latestTeams[name];
    if (!rec) continue;
    anyTeams = true;
    const p = rec.p;
    const br = computeBurnRate(entries, name);
    const pad = name.padEnd(16);

    if (p.status === 'invalid_key') {
      lines.push(`  ${pad} 401 invalid key`);
    } else if (p.status === 'rejected') {
      lines.push(`  ${pad} MAXED — resets in ${formatDuration(p.reset_in_seconds)}`);
    } else if (p.utilization_7d == null && p.utilization_5h == null) {
      lines.push(`  ${pad} DORMANT — cycle not started`);
    } else if (br && br.data_points >= 2) {
      const rateStr = `${(br.rate_per_hour * 100).toFixed(1)}%/hr`;
      const exhStr = br.projected_exhaustion_hours != null
        ? `exhausts ~${Math.round(br.projected_exhaustion_hours)}h`
        : 'stable/declining';
      lines.push(`  ${pad} 7d: ${pct(br.first_util_7d)}→${pct(br.current_util_7d)} over ${br.hours_elapsed.toFixed(1)}h = ${rateStr} | ${exhStr} | ${br.data_points} pts`);
    } else {
      lines.push(`  ${pad} 7d: ${pct(p.utilization_7d)} (insufficient data for rate)`);
    }
  }
  if (!anyTeams) lines.push('  (no teams data in logs)');
  lines.push('');

  // ── Reset schedule
  const stagger = cycleStagger(entries);
  if (stagger.length > 0) {
    lines.push('Reset Schedule (next 48h)');
    for (const { provider, resets_at_iso, resets_in_seconds_from_now } of stagger) {
      const timeStr = resets_at_iso.slice(11, 16) + ' UTC';
      lines.push(`  ${provider.padEnd(16)} ~${formatDuration(resets_in_seconds_from_now).padEnd(10)} (${timeStr})`);
    }
    lines.push('');
  }

  // ── Weekly
  const weekly = reconstructWeekly(entries);
  const weekKeys = Object.keys(weekly).sort();
  if (weekKeys.length > 0) {
    lines.push('Weekly Reconstruction');
    for (const week of weekKeys) {
      const w = weekly[week];
      const note = w.start === w.end ? ' (1 day)' : '';
      lines.push(`  ${week}${note}`);
      for (const [name, s] of Object.entries(w.providers)) {
        const exhStr = s.exhausted_count > 0 ? ` | exhausted: ${s.exhausted_count}x` : '';
        lines.push(`    ${name.padEnd(14)} peak 7d: ${pct(s.peak_util_7d)} | avg: ${pct(s.avg_util_7d)} | ${s.samples} samples${exhStr}`);
      }
    }
    lines.push('');
  }

  // ── Rotation
  const rotation = rotationRank(entries);
  if (rotation.length > 0) {
    lines.push('Rotation Recommendation');
    rotation.forEach(({ provider, reason, severity }, i) => {
      const icon = severity === 'ok' ? '✓' : severity === 'critical' ? '✗' : severity === 'dormant' ? '~' : '?';
      lines.push(`  ${i + 1}. ${provider.padEnd(16)} ${icon} ${reason}`);
    });
    lines.push('');
  }

  // ── Underspend alerts
  const boosts = underspendAlerts(entries);
  if (boosts.length > 0) {
    lines.push('⚡ Underspend Alerts (burn before reset)');
    for (const { provider, utilization_5h, reset_in_seconds } of boosts) {
      lines.push(`  ${provider}: 5h at ${pct(utilization_5h)}, resets in ${formatDuration(reset_in_seconds)}`);
    }
    lines.push('');
  }

  if (entries.length === 0) {
    lines.push('No log data found. Run monitor.js to start accumulating data.');
  }

  return lines.join('\n');
}

// ── Main ──────────────────────────────────────────────────────────────────────

const args = process.argv.slice(2);
const showBurnRate  = args.includes('--burn-rate');
const showWeekly    = args.includes('--weekly');
const showStagger   = args.includes('--stagger');
const showRotation  = args.includes('--rotation');
const isJson        = args.includes('--json');
const isPrune       = args.includes('--prune');
const isDryRun      = args.includes('--dry-run');
const providerIdx   = args.indexOf('--provider');
const providerFilter = providerIdx !== -1 ? args[providerIdx + 1] : null;
const showAll = !showBurnRate && !showWeekly && !showStagger && !showRotation && !isPrune;

if (isPrune) {
  pruneLogs(isDryRun);
  process.exit(0);
}

const entries = loadLogs(providerFilter);

if (isJson) {
  const burnRates = {};
  for (const name of TEAMS) {
    const br = computeBurnRate(entries, name);
    if (br) burnRates[name] = br;
  }
  console.log(JSON.stringify({
    timestamp: new Date().toISOString(),
    burn_rates: burnRates,
    weekly: reconstructWeekly(entries),
    stagger: cycleStagger(entries),
    rotation: rotationRank(entries),
    underspend_alerts: underspendAlerts(entries),
  }, null, 2));
  process.exit(0);
}

if (showAll) {
  console.log(generateFullReport(entries));
  process.exit(0);
}

// Section-specific output
const width = 56;
if (showBurnRate) {
  console.log('Burn Rate\n' + '─'.repeat(width));
  const latestTeams = latestPerProvider(entries, 'teams-direct');
  for (const name of TEAMS) {
    const rec = latestTeams[name];
    if (!rec) continue;
    const br = computeBurnRate(entries, name);
    if (br && br.data_points >= 2) {
      const exhStr = br.projected_exhaustion_hours != null
        ? `exhausts ~${Math.round(br.projected_exhaustion_hours)}h`
        : 'stable/declining';
      console.log(`  ${name.padEnd(16)} ${(br.rate_per_hour * 100).toFixed(1)}%/hr | ${exhStr} | ${br.data_points} pts`);
    } else {
      console.log(`  ${name.padEnd(16)} insufficient data`);
    }
  }
}

if (showWeekly) {
  console.log('Weekly Reconstruction\n' + '─'.repeat(width));
  const weekly = reconstructWeekly(entries);
  for (const [week, w] of Object.entries(weekly).sort()) {
    console.log(`  ${week}`);
    for (const [name, s] of Object.entries(w.providers)) {
      console.log(`    ${name.padEnd(14)} peak 7d: ${pct(s.peak_util_7d)} | avg: ${pct(s.avg_util_7d)} | ${s.samples} samples`);
    }
  }
}

if (showStagger) {
  console.log('Reset Schedule (next 48h)\n' + '─'.repeat(width));
  for (const { provider, resets_in_seconds_from_now, resets_at_iso } of cycleStagger(entries)) {
    const timeStr = resets_at_iso.slice(11, 16) + ' UTC';
    console.log(`  ${provider.padEnd(16)} ~${formatDuration(resets_in_seconds_from_now)} (${timeStr})`);
  }
}

if (showRotation) {
  console.log('Rotation Recommendation\n' + '─'.repeat(width));
  rotationRank(entries).forEach(({ provider, reason, severity }, i) => {
    const icon = severity === 'ok' ? '✓' : severity === 'critical' ? '✗' : '~';
    console.log(`  ${i + 1}. ${provider.padEnd(16)} ${icon} ${reason}`);
  });
}