diff --git a/README.md b/README.md index eda68da..799e9fa 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,157 @@ # token-monitor -Modular LLM API quota and usage visibility tool \ No newline at end of file +Modular LLM API quota and usage visibility tool. Extracts rate-limit and usage data from all configured LLM providers before failures happen. + +**Why it exists:** team-vigilio hit its 7-day rate limit (9+ days of 429s). api-ateam ran out of credit mid-session. We kept flying blind. This tool surfaces quota health before the failure. + +## Usage + +```bash +node monitor.js # human-readable summary + log to file +node monitor.js --json # JSON output + log to file +node monitor.js --summary # human-readable only (no log) +node monitor.js --provider team-nadja # single provider +node monitor.js --no-log # suppress log file write +``` + +## Example output + +``` +Token Monitor — 2026-04-04 16:59 UTC +════════════════════════════════════════════════════════════ + +team-vigilio [CRITICAL] MAXED — 7d: 100% | resets in 23h 0m +team-ludo [UNKNOWN] Invalid API key (401) +team-molto [OK] 5h: 32% | 7d: 45% | resets in 4h 0m +team-nadja [OK] 5h: 52% | 7d: 39% | resets in 3h 0m +team-buio [UNKNOWN] Invalid API key (401) +shelley-proxy [OK] tokens: 4,800,000/4,800,000 | cost: $0.000013/call +api-ateam [UNKNOWN] Anthropic API does not expose billing/quota via REST + +──────────────────────────────────────────────────────────── +Overall: 1 CRITICAL, 3 OK, 3 UNKNOWN +``` + +## JSON output schema + +```json +{ + "timestamp": "2026-04-04T16:59:50.000Z", + "providers": { + "team-vigilio": { + "type": "teams-direct", + "status": "rejected", + "utilization_5h": 0.94, + "utilization_7d": 1.0, + "representative_claim": "seven_day", + "reset_timestamp": 1743800000, + "reset_in_seconds": 82800, + "organization_id": "1d7653ad-...", + "severity": "critical", + "probe_latency_ms": 210 + }, + "shelley-proxy": { + "type": "shelley-proxy", + "status": "ok", + "tokens_remaining": 4800000, + "tokens_limit": 4800000, + "requests_remaining": 20000, + "requests_limit": 20000, + "tokens_reset": "2026-04-04T17:59:50Z", + "cost_per_call_usd": 0.000013, + "severity": "ok", + "probe_latency_ms": 180 + }, + "api-ateam": { + "type": "api-direct", + "status": "no_billing_data", + "message": "Anthropic API does not expose billing/quota via REST.", + "severity": "unknown", + "probe_latency_ms": 0 + } + } +} +``` + +## Provider support + +| Provider | Type | Data available | +|----------|------|---------------| +| team-vigilio, team-ludo, team-molto, team-nadja, team-buio | Anthropic Teams (direct) | 5h/7d utilization (0–100%), status, reset countdown, severity | +| shelley-proxy | Shelley/exe.dev proxy | Token headroom, request headroom, per-call USD cost | +| api-ateam | Anthropic API (pay-per-use) | Key validity only — no billing API exists | + +## Severity levels + +| Level | Condition | +|-------|-----------| +| `critical` | Teams provider rejected (rate limited / budget exhausted) | +| `warning` | Teams 7d utilization > 85%, or 5h utilization > 70% | +| `ok` | Healthy | +| `unknown` | Invalid key (401), no billing data, or probe error | + +## Rate-limit header schemas + +**Teams direct** (oat01 keys — all `team-*` providers): +Anthropic Teams uses the **unified** schema. Headers are present on every response (200 and 429). +- `anthropic-ratelimit-unified-5h-utilization` — 5-hour window utilization (0.0–1.0) +- `anthropic-ratelimit-unified-7d-utilization` — 7-day budget utilization (0.0–1.0) +- `anthropic-ratelimit-unified-status` — `allowed` | `rejected` +- `anthropic-ratelimit-unified-representative-claim` — which window is binding (`five_hour` | `seven_day`) +- `anthropic-ratelimit-unified-reset` — Unix timestamp when binding window resets + +**Shelley proxy** (exe.dev gateway): +Uses classic absolute-count headers plus an exe.dev-injected cost header: +- `Anthropic-Ratelimit-Tokens-Remaining` / `Limit` — absolute token counts +- `Anthropic-Ratelimit-Requests-Remaining` / `Limit` — request counts +- `Exedev-Gateway-Cost` — USD cost per call (unique to exe.dev gateway) + +## Log files + +Each run appends to `~/.logs/token-monitor/YYYY-MM-DD.jsonl`: + +```bash +# View today's log +cat ~/.logs/token-monitor/$(date +%Y-%m-%d).jsonl | \ + python3 -c "import sys,json; [print(json.loads(l)['ts'], json.loads(l)['providers']['team-vigilio']['severity']) for l in sys.stdin]" + +# Check when team-vigilio was last healthy +grep '"team-vigilio"' ~/.logs/token-monitor/*.jsonl | \ + python3 -c "import sys,json; [print(l[:40]) for l in sys.stdin if json.loads(l.split(':',1)[1])['providers']['team-vigilio']['severity']=='ok']" +``` + +## Architecture + +``` +monitor.js — CLI entrypoint, orchestrates probes +providers/ + index.js — reads ~/.pi/agent/models.json, returns typed provider list + anthropic-teams.js — unified schema parser (oat01 keys, all team-* providers) + anthropic-api.js — pay-per-use (api03 keys) — reports "no billing data" + shelley-proxy.js — classic schema + Exedev-Gateway-Cost header +logger.js — JSONL log to ~/.logs/token-monitor/ +report.js — human-readable summary + severity logic +test.js — test suite (run: node test.js) +``` + +## How to add a new provider + +1. Add provider to `~/.pi/agent/models.json` with `"api": "anthropic-messages"` +2. If it uses Teams unified schema and its name starts with `team-`: picked up automatically +3. For a custom schema: create `providers/yourprovider.js`, implement `probeYourProvider()`, classify it in `providers/index.js`, add probe dispatch in `monitor.js` + +## Probe behavior + +The tool makes one minimal API call per provider to extract headers: +- Model: `claude-haiku-4-5-20251001` (cheapest available) +- Max tokens: 1 (minimizes cost) +- Rate-limit headers are returned on **every** response (200 and 429) for Teams providers +- A 429 from a maxed provider is expected and treated as valid quota data + +**Stealth:** Read-only, one call per run, no hammering. Run at most once per session. + +## Related + +- `~/projects/provider-check/` — predecessor (liveness only, no quota depth) +- `~/.pi/agent/models.json` — provider configuration source +- Forgejo issue: trentuna/a-team#91 diff --git a/logger.js b/logger.js new file mode 100644 index 0000000..7fd8b7c --- /dev/null +++ b/logger.js @@ -0,0 +1,19 @@ +/** + * logger.js — persistent JSONL log to ~/.logs/token-monitor/YYYY-MM-DD.jsonl + */ + +import { appendFileSync, mkdirSync } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; + +export function logRun(data) { + const dir = join(homedir(), '.logs', 'token-monitor'); + mkdirSync(dir, { recursive: true }); + const file = join(dir, `${new Date().toISOString().slice(0, 10)}.jsonl`); + appendFileSync(file, JSON.stringify({ ts: new Date().toISOString(), ...data }) + '\n'); +} + +export function getLogPath() { + const today = new Date().toISOString().slice(0, 10); + return join(homedir(), '.logs', 'token-monitor', `${today}.jsonl`); +} diff --git a/monitor.js b/monitor.js new file mode 100644 index 0000000..d78558a --- /dev/null +++ b/monitor.js @@ -0,0 +1,93 @@ +#!/usr/bin/env node +/** + * monitor.js — Token Monitor entrypoint + * + * Probes all configured LLM API providers, extracts rate-limit/quota headers, + * and outputs a human-readable summary or JSON. + * + * Usage: + * node monitor.js # human-readable summary + log + * node monitor.js --json # JSON to stdout + log + * node monitor.js --summary # human-readable only (no log) + * node monitor.js --provider team-nadja # single provider + * node monitor.js --no-log # suppress log file + */ + +import { getProviders } from './providers/index.js'; +import { probeTeamsProvider } from './providers/anthropic-teams.js'; +import { getApiAteamStatus } from './providers/anthropic-api.js'; +import { probeShelleyProxy } from './providers/shelley-proxy.js'; +import { generateReport, getSeverity } from './report.js'; +import { logRun } from './logger.js'; + +const args = process.argv.slice(2); +const isJson = args.includes('--json'); +const isSummaryOnly = args.includes('--summary'); +const noLog = args.includes('--no-log'); +const filterIdx = args.indexOf('--provider'); +const filterProvider = filterIdx !== -1 ? args[filterIdx + 1] : null; + +/** + * Probe a single provider and return normalized result. + */ +async function probeProvider(p) { + const start = Date.now(); + let result; + + if (p.type === 'teams-direct') { + result = await probeTeamsProvider(p.name, p.baseUrl, p.apiKey); + } else if (p.type === 'shelley-proxy') { + result = await probeShelleyProxy(p.name, p.baseUrl); + } else if (p.type === 'api-direct') { + result = getApiAteamStatus(); + } else { + result = { type: 'unknown', status: 'skipped', severity: 'unknown' }; + } + + result.probe_latency_ms = Date.now() - start; + if (!result.severity) { + result.severity = getSeverity(result); + } + return result; +} + +async function main() { + const allProviders = getProviders(); + const providerNames = filterProvider + ? [filterProvider] + : Object.keys(allProviders); + + if (filterProvider && !allProviders[filterProvider]) { + console.error( + `Unknown provider: ${filterProvider}. Available: ${Object.keys(allProviders).join(', ')}` + ); + process.exit(1); + } + + const results = {}; + for (const name of providerNames) { + const p = allProviders[name]; + results[name] = await probeProvider(p); + } + + const output = { + timestamp: new Date().toISOString(), + providers: results, + }; + + if (isJson) { + console.log(JSON.stringify(output, null, 2)); + } else { + console.log(generateReport(output)); + } + + // Log to file unless --summary or --no-log + if (!isSummaryOnly && !noLog) { + logRun(output); + } +} + +main().catch((err) => { + console.error('Fatal:', err.message); + process.exit(1); +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..f954abb --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "token-monitor", + "version": "0.1.0", + "description": "Modular LLM API quota and usage visibility tool", + "type": "module", + "main": "monitor.js", + "scripts": { + "start": "node monitor.js", + "check": "node monitor.js --summary", + "test": "node test.js" + }, + "engines": { + "node": ">=18" + } +} diff --git a/providers/anthropic-api.js b/providers/anthropic-api.js new file mode 100644 index 0000000..8555168 --- /dev/null +++ b/providers/anthropic-api.js @@ -0,0 +1,75 @@ +/** + * anthropic-api.js — api-ateam (pay-per-use, api03 keys) + * + * Anthropic's REST API does not expose billing or quota data via any endpoint + * (confirmed by recon, issue trentuna/a-team#91). This module reports that + * billing data is unavailable — always. Key validity could be confirmed with + * a real HTTP call, but for monitoring purposes the static result is authoritative. + */ + +import { getSeverity } from '../report.js'; + +/** + * Return the static api-ateam status object. + * Synchronous — no HTTP call needed because no billing API exists. + * @returns {Object} + */ +export function getApiAteamStatus() { + return { + type: 'api-direct', + status: 'no_billing_data', + message: 'Anthropic API does not expose billing/quota via REST. Key validity not checked.', + severity: getSeverity({ type: 'api-direct' }), + }; +} + +/** + * Probe the api-ateam provider. + * Makes a minimal call to confirm key validity, but always reports no billing data. + * @param {string} providerName + * @param {string} baseUrl + * @param {string} apiKey + * @returns {Promise} + */ +export async function probeApiProvider(providerName, baseUrl, apiKey) { + try { + const response = await fetch(`${baseUrl}/v1/messages`, { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 1, + messages: [{ role: 'user', content: 'Hi' }], + }), + }); + + if (response.status === 401) { + return { + type: 'api-direct', + status: 'invalid_key', + message: 'Invalid API key (401)', + severity: 'unknown', + }; + } + + // No billing endpoint exists — always report no_billing_data + return { + type: 'api-direct', + status: 'no_billing_data', + message: 'Anthropic API does not expose billing/quota via REST. Key appears valid.', + http_status: response.status, + severity: 'unknown', + }; + } catch (err) { + return { + type: 'api-direct', + status: 'error', + message: err.message, + severity: 'unknown', + }; + } +} diff --git a/providers/anthropic-teams.js b/providers/anthropic-teams.js new file mode 100644 index 0000000..3a0991f --- /dev/null +++ b/providers/anthropic-teams.js @@ -0,0 +1,130 @@ +/** + * anthropic-teams.js — Unified schema parser for Anthropic Teams direct providers. + * + * Teams providers (team-vigilio, team-ludo, team-molto, team-nadja, team-buio) use the + * anthropic-ratelimit-unified-* header family. Headers are present on EVERY response + * (200 and 429). A 429 is expected when the 7d budget is exhausted and contains valid + * quota data — do not treat it as an error, extract the headers normally. + * + * Header reference (from Face's recon, issue trentuna/a-team#91): + * anthropic-ratelimit-unified-status allowed|rejected + * anthropic-ratelimit-unified-5h-status allowed|rejected + * anthropic-ratelimit-unified-5h-utilization 0.0–1.0 + * anthropic-ratelimit-unified-5h-reset Unix timestamp + * anthropic-ratelimit-unified-7d-status allowed|rejected + * anthropic-ratelimit-unified-7d-utilization 0.0–1.0 + * anthropic-ratelimit-unified-7d-reset Unix timestamp + * anthropic-ratelimit-unified-7d-surpassed-threshold (present only when maxed) + * anthropic-ratelimit-unified-representative-claim five_hour|seven_day + * anthropic-ratelimit-unified-fallback-percentage 0.0–1.0 + * anthropic-ratelimit-unified-reset Unix timestamp (binding reset) + * anthropic-ratelimit-unified-overage-status rejected + * anthropic-ratelimit-unified-overage-disabled-reason org_level_disabled + * anthropic-organization-id UUID + * retry-after seconds (only on 429) + */ + +import { getSeverity } from '../report.js'; + +/** + * Parse unified rate-limit headers from a Teams API response. + * + * @param {Object} headers — fetch Response.headers (or compatible mock with .get(name)) + * @param {number} httpStatus — HTTP status code of the response + * @param {string} providerName — name for logging/context + * @returns {Object} normalized provider result + */ +export function parseTeamsHeaders(headers, httpStatus, providerName) { + const h = (name) => headers.get(name); + + // 401 = invalid API key — no quota data available + if (httpStatus === 401) { + return { + type: 'teams-direct', + status: 'invalid_key', + utilization_5h: null, + utilization_7d: null, + severity: 'unknown', + }; + } + + const status = h('anthropic-ratelimit-unified-status') || (httpStatus === 429 ? 'rejected' : 'allowed'); + const util5h = parseFloat(h('anthropic-ratelimit-unified-5h-utilization')); + const util7d = parseFloat(h('anthropic-ratelimit-unified-7d-utilization')); + const resetTs = parseInt(h('anthropic-ratelimit-unified-reset'), 10); + const retryAfter = h('retry-after') ? parseInt(h('retry-after'), 10) : null; + + // Compute reset_in_seconds from the binding reset Unix timestamp + const nowSec = Math.floor(Date.now() / 1000); + const resetInSeconds = !isNaN(resetTs) ? Math.max(0, resetTs - nowSec) : null; + + const result = { + type: 'teams-direct', + status, + utilization_5h: isNaN(util5h) ? null : util5h, + utilization_7d: isNaN(util7d) ? null : util7d, + representative_claim: h('anthropic-ratelimit-unified-representative-claim') || null, + reset_timestamp: isNaN(resetTs) ? null : resetTs, + reset_in_seconds: resetInSeconds, + organization_id: h('anthropic-organization-id') || null, + // Additional detail headers + status_5h: h('anthropic-ratelimit-unified-5h-status') || null, + status_7d: h('anthropic-ratelimit-unified-7d-status') || null, + overage_status: h('anthropic-ratelimit-unified-overage-status') || null, + fallback_percentage: h('anthropic-ratelimit-unified-fallback-percentage') + ? parseFloat(h('anthropic-ratelimit-unified-fallback-percentage')) + : null, + }; + + // Include retry_after_seconds only when present (429 responses) + if (retryAfter !== null) { + result.retry_after_seconds = retryAfter; + } + + // Include surpassed threshold when present (maxed budget) + const surpassed = h('anthropic-ratelimit-unified-7d-surpassed-threshold'); + if (surpassed !== null) { + result.surpassed_threshold_7d = parseFloat(surpassed); + } + + result.severity = getSeverity(result); + return result; +} + +/** + * Probe a single Teams provider by making a minimal API call. + * Extracts headers regardless of whether the response is 200 or 429. + * + * @param {string} providerName + * @param {string} baseUrl + * @param {string} apiKey + * @returns {Promise} normalized provider result + */ +export async function probeTeamsProvider(providerName, baseUrl, apiKey) { + try { + const response = await fetch(`${baseUrl}/v1/messages`, { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 1, + messages: [{ role: 'user', content: 'Hi' }], + }), + }); + + return parseTeamsHeaders(response.headers, response.status, providerName); + } catch (err) { + return { + type: 'teams-direct', + status: 'error', + message: err.message, + utilization_5h: null, + utilization_7d: null, + severity: 'unknown', + }; + } +} diff --git a/providers/index.js b/providers/index.js new file mode 100644 index 0000000..dff64ed --- /dev/null +++ b/providers/index.js @@ -0,0 +1,47 @@ +/** + * providers/index.js — provider registry + * + * Reads ~/.pi/agent/models.json and returns typed provider config for all + * providers we know how to probe. Non-anthropic-messages providers (e.g. zai) + * are silently skipped. + */ + +import { readFileSync } from 'fs'; +import { homedir } from 'os'; + +/** + * Classify a provider by name and config. + * @returns {'teams-direct'|'shelley-proxy'|'api-direct'|null} + */ +function classifyProvider(name, config) { + if (name === 'shelley-proxy') return 'shelley-proxy'; + if (name === 'api-ateam') return 'api-direct'; + if (config.api === 'anthropic-messages' && name.startsWith('team-')) return 'teams-direct'; + return null; // skip (zai, etc.) +} + +/** + * Load and classify all providers from models.json. + * @returns {Object} map of provider name → { name, type, baseUrl, apiKey } + */ +export function getProviders() { + const modelsJson = JSON.parse( + readFileSync(`${homedir()}/.pi/agent/models.json`, 'utf-8') + ); + + const providers = {}; + for (const [name, config] of Object.entries(modelsJson.providers)) { + const type = classifyProvider(name, config); + if (!type) continue; + providers[name] = { + name, + type, + baseUrl: config.baseUrl, + apiKey: config.apiKey || null, + }; + } + return providers; +} + +// Alias for backwards compatibility +export const loadProviders = getProviders; diff --git a/providers/shelley-proxy.js b/providers/shelley-proxy.js new file mode 100644 index 0000000..f3b20ae --- /dev/null +++ b/providers/shelley-proxy.js @@ -0,0 +1,96 @@ +/** + * shelley-proxy.js — Shelley/exe.dev proxy (classic schema + Exedev-Gateway-Cost) + * + * The Shelley proxy returns standard Anthropic rate-limit headers (classic schema, + * not the unified Teams schema) plus an Exedev-Gateway-Cost header with per-call + * USD cost. No API key is required — the proxy handles auth internally. + * + * Header reference: + * Anthropic-Ratelimit-Tokens-Limit total token budget + * Anthropic-Ratelimit-Tokens-Remaining remaining tokens + * Anthropic-Ratelimit-Tokens-Reset ISO 8601 reset time + * Anthropic-Ratelimit-Requests-Limit total request budget + * Anthropic-Ratelimit-Requests-Remaining remaining requests + * Anthropic-Ratelimit-Requests-Reset ISO 8601 reset time + * Exedev-Gateway-Cost per-call USD cost (float) + * anthropic-organization-id organization UUID + */ + +import { getSeverity } from '../report.js'; + +/** + * Parse classic Anthropic rate-limit headers from a Shelley proxy response. + * + * @param {Object} headers — fetch Response.headers (or compatible mock with .get(name)) + * @param {number} httpStatus — HTTP status code + * @returns {Object} normalized provider result + */ +export function parseShelleyHeaders(headers, httpStatus) { + const h = (name) => headers.get(name) || headers.get(name.toLowerCase()); + + const tokensLimit = parseInt(h('Anthropic-Ratelimit-Tokens-Limit'), 10); + const tokensRemaining = parseInt(h('Anthropic-Ratelimit-Tokens-Remaining'), 10); + const tokensReset = h('Anthropic-Ratelimit-Tokens-Reset'); + const requestsLimit = parseInt(h('Anthropic-Ratelimit-Requests-Limit'), 10); + const requestsRemaining = parseInt(h('Anthropic-Ratelimit-Requests-Remaining'), 10); + const requestsReset = h('Anthropic-Ratelimit-Requests-Reset'); + const costPerCall = h('Exedev-Gateway-Cost'); + const orgId = h('anthropic-organization-id'); + + const result = { + type: 'shelley-proxy', + status: httpStatus === 429 ? 'rate_limited' : (httpStatus === 200 ? 'ok' : 'error'), + tokens_limit: isNaN(tokensLimit) ? null : tokensLimit, + tokens_remaining: isNaN(tokensRemaining) ? null : tokensRemaining, + tokens_reset: tokensReset || null, + requests_limit: isNaN(requestsLimit) ? null : requestsLimit, + requests_remaining: isNaN(requestsRemaining) ? null : requestsRemaining, + requests_reset: requestsReset || null, + cost_per_call_usd: costPerCall ? parseFloat(costPerCall) : null, + organization_id: orgId || null, + }; + + result.severity = getSeverity(result); + return result; +} + +// Alias used in some internal tooling +export const parseClassicHeaders = parseShelleyHeaders; + +/** + * Probe the Shelley proxy by making a minimal API call. + * @param {string} providerName + * @param {string} baseUrl + * @returns {Promise} normalized provider result + */ +export async function probeShelleyProxy(providerName, baseUrl) { + try { + const response = await fetch(`${baseUrl}/v1/messages`, { + method: 'POST', + headers: { + 'x-api-key': 'not-needed', + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 1, + messages: [{ role: 'user', content: 'Hi' }], + }), + }); + + return parseShelleyHeaders(response.headers, response.status); + } catch (err) { + return { + type: 'shelley-proxy', + status: 'error', + message: err.message, + tokens_limit: null, + tokens_remaining: null, + requests_limit: null, + requests_remaining: null, + cost_per_call_usd: null, + severity: 'unknown', + }; + } +} diff --git a/report.js b/report.js new file mode 100644 index 0000000..06d7897 --- /dev/null +++ b/report.js @@ -0,0 +1,128 @@ +/** + * report.js — human-readable summary generator + severity logic + */ + +/** + * Compute severity for a parsed provider result. + * @param {Object} provider + * @returns {'critical'|'warning'|'ok'|'unknown'} + */ +export function getSeverity(provider) { + if (provider.type === 'teams-direct') { + if (provider.status === 'rejected') return 'critical'; + if (provider.utilization_7d > 0.85) return 'warning'; + if (provider.utilization_5h > 0.7) return 'warning'; + return 'ok'; + } + if (provider.type === 'shelley-proxy') { + const tokenPct = 1 - (provider.tokens_remaining / provider.tokens_limit); + if (tokenPct > 0.85) return 'warning'; + return 'ok'; + } + return 'unknown'; +} + +/** + * Format seconds as "Xh Ym" or "Xm" or "Xs". + */ +function formatDuration(seconds) { + if (seconds == null || isNaN(seconds) || seconds < 0) return '?'; + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + const s = seconds % 60; + if (h > 0) return `${h}h ${m}m`; + if (m > 0) return `${m}m`; + return `${s}s`; +} + +/** + * Format a float as a percentage string. + */ +function pct(v) { + if (v == null) return '?'; + return `${Math.round(v * 100)}%`; +} + +/** + * Severity badge string. + */ +function badge(severity) { + switch (severity) { + case 'critical': return '[CRITICAL]'; + case 'warning': return '[WARNING] '; + case 'ok': return '[OK] '; + default: return '[UNKNOWN] '; + } +} + +/** + * Generate a human-readable summary from a full monitor result. + * @param {Object} result — { timestamp, providers: { name: {...} } } + * @returns {string} + */ +export function generateReport(result) { + const ts = result.timestamp + ? result.timestamp.replace('T', ' ').replace(/\.\d+Z$/, ' UTC').replace('Z', ' UTC') + : new Date().toUTCString(); + + const lines = []; + const width = 60; + + lines.push(`Token Monitor — ${ts}`); + lines.push('═'.repeat(width)); + lines.push(''); + + const counts = { critical: 0, warning: 0, ok: 0, unknown: 0 }; + + for (const [name, p] of Object.entries(result.providers)) { + const sev = p.severity || getSeverity(p); + counts[sev] = (counts[sev] || 0) + 1; + + const b = badge(sev); + let detail = ''; + + if (p.type === 'teams-direct') { + if (p.status === 'invalid_key') { + detail = 'Invalid API key (401)'; + } else if (p.status === 'rejected') { + const resetIn = formatDuration(p.reset_in_seconds); + detail = `MAXED — 7d: ${pct(p.utilization_7d)} | resets in ${resetIn}`; + } else { + detail = `5h: ${pct(p.utilization_5h)} | 7d: ${pct(p.utilization_7d)}`; + if (p.reset_in_seconds != null) { + detail += ` | resets in ${formatDuration(p.reset_in_seconds)}`; + } + } + } else if (p.type === 'shelley-proxy') { + if (p.status === 'error') { + detail = `Error: ${p.message || 'unknown'}`; + } else { + detail = `tokens: ${p.tokens_remaining?.toLocaleString()}/${p.tokens_limit?.toLocaleString()}`; + if (p.cost_per_call_usd != null) { + detail += ` | cost: $${p.cost_per_call_usd}/call`; + } + } + } else if (p.type === 'api-direct') { + detail = p.message || 'No billing data available'; + } else { + detail = p.message || ''; + } + + // Pad provider name to 14 chars + const paddedName = name.padEnd(14); + lines.push(`${paddedName} ${b} ${detail}`); + } + + lines.push(''); + lines.push('─'.repeat(width)); + + const parts = []; + if (counts.critical) parts.push(`${counts.critical} CRITICAL`); + if (counts.warning) parts.push(`${counts.warning} WARNING`); + if (counts.ok) parts.push(`${counts.ok} OK`); + if (counts.unknown) parts.push(`${counts.unknown} UNKNOWN`); + lines.push(`Overall: ${parts.join(', ') || 'no providers'}`); + lines.push(''); + + return lines.join('\n'); +} diff --git a/test.js b/test.js new file mode 100644 index 0000000..9b85ee3 --- /dev/null +++ b/test.js @@ -0,0 +1,335 @@ +/** + * token-monitor test suite + * Run: node test.js + * Red first, then implement until green. + */ + +import { readFileSync, existsSync } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; +import { execSync } from 'child_process'; + +let passed = 0; +let failed = 0; +const failures = []; + +function assert(label, condition, detail = '') { + if (condition) { + console.log(` ✓ ${label}`); + passed++; + } else { + console.log(` ✗ ${label}${detail ? ': ' + detail : ''}`); + failed++; + failures.push({ label, detail }); + } +} + +function run(cmd) { + return execSync(cmd, { cwd: '/home/exedev/projects/token-monitor', encoding: 'utf-8' }); +} + +function runSafe(cmd) { + try { + return { stdout: run(cmd), code: 0 }; + } catch (e) { + return { stdout: e.stdout || '', stderr: e.stderr || '', code: e.status }; + } +} + +// ── 1. Package structure ───────────────────────────────────────────────────── +console.log('\n── 1. File structure ───────────────────────────────────────────'); + +const root = '/home/exedev/projects/token-monitor'; +const files = [ + 'package.json', + 'monitor.js', + 'logger.js', + 'report.js', + 'providers/index.js', + 'providers/anthropic-teams.js', + 'providers/anthropic-api.js', + 'providers/shelley-proxy.js', + 'README.md', +]; +for (const f of files) { + assert(`${f} exists`, existsSync(join(root, f))); +} + +// ── 2. package.json shape ──────────────────────────────────────────────────── +console.log('\n── 2. package.json ─────────────────────────────────────────────'); +const pkg = JSON.parse(readFileSync(join(root, 'package.json'), 'utf-8')); +assert('name = token-monitor', pkg.name === 'token-monitor'); +assert('type = module (ESM)', pkg.type === 'module'); +assert('no external dependencies', !pkg.dependencies || Object.keys(pkg.dependencies).length === 0); +assert('engines.node >= 18', pkg.engines?.node?.includes('18') || pkg.engines?.node?.includes('>=18')); + +// ── 3. Severity logic (unit test, importable) ──────────────────────────────── +console.log('\n── 3. Severity logic ───────────────────────────────────────────'); +const { getSeverity } = await import('./report.js'); + +assert('teams: rejected → critical', + getSeverity({ type: 'teams-direct', status: 'rejected', utilization_5h: 0, utilization_7d: 1.0 }) === 'critical'); +assert('teams: 7d > 0.85 → warning', + getSeverity({ type: 'teams-direct', status: 'allowed', utilization_5h: 0.2, utilization_7d: 0.9 }) === 'warning'); +assert('teams: 5h > 0.7 → warning', + getSeverity({ type: 'teams-direct', status: 'allowed', utilization_5h: 0.75, utilization_7d: 0.4 }) === 'warning'); +assert('teams: healthy → ok', + getSeverity({ type: 'teams-direct', status: 'allowed', utilization_5h: 0.3, utilization_7d: 0.4 }) === 'ok'); +assert('shelley: high token use → warning', + getSeverity({ type: 'shelley-proxy', tokens_remaining: 5000, tokens_limit: 50000 }) === 'warning'); +assert('shelley: healthy → ok', + getSeverity({ type: 'shelley-proxy', tokens_remaining: 45000, tokens_limit: 50000 }) === 'ok'); +assert('api-direct → unknown', + getSeverity({ type: 'api-direct' }) === 'unknown'); + +// ── 4. Provider registry ───────────────────────────────────────────────────── +console.log('\n── 4. Provider registry ────────────────────────────────────────'); +const { getProviders } = await import('./providers/index.js'); +const providers = getProviders(); +const names = Object.keys(providers); +assert('team-vigilio in registry', names.includes('team-vigilio')); +assert('team-molto in registry', names.includes('team-molto')); +assert('team-nadja in registry', names.includes('team-nadja')); +assert('team-ludo in registry', names.includes('team-ludo')); +assert('team-buio in registry', names.includes('team-buio')); +assert('shelley-proxy in registry', names.includes('shelley-proxy')); +assert('api-ateam in registry', names.includes('api-ateam')); +assert('zai NOT in registry (not anthropic-messages)', !names.includes('zai')); +for (const [name, p] of Object.entries(providers)) { + assert(`${name} has baseUrl`, typeof p.baseUrl === 'string' && p.baseUrl.length > 0); + assert(`${name} has type`, ['teams-direct', 'shelley-proxy', 'api-direct'].includes(p.type)); +} + +// ── 5. Teams header parser ─────────────────────────────────────────────────── +console.log('\n── 5. Teams header parser ──────────────────────────────────────'); +const { parseTeamsHeaders } = await import('./providers/anthropic-teams.js'); + +// Simulate a 200 OK response with unified headers +const ok200headers = new Map([ + ['anthropic-ratelimit-unified-status', 'allowed'], + ['anthropic-ratelimit-unified-5h-status', 'allowed'], + ['anthropic-ratelimit-unified-5h-utilization', '0.32'], + ['anthropic-ratelimit-unified-5h-reset', '1775336400'], + ['anthropic-ratelimit-unified-7d-status', 'allowed'], + ['anthropic-ratelimit-unified-7d-utilization', '0.45'], + ['anthropic-ratelimit-unified-7d-reset', '1775869200'], + ['anthropic-ratelimit-unified-representative-claim', 'five_hour'], + ['anthropic-ratelimit-unified-fallback-percentage', '0.5'], + ['anthropic-ratelimit-unified-reset', '1775336400'], + ['anthropic-ratelimit-unified-overage-status', 'rejected'], + ['anthropic-ratelimit-unified-overage-disabled-reason', 'org_level_disabled'], + ['anthropic-organization-id', '1d7653ad-11d9-4029-ba1a-a2dd4cd0b2f3'], +]); +const fakeHeaders200 = { get: (k) => ok200headers.get(k.toLowerCase()) || null }; +const parsed200 = parseTeamsHeaders(fakeHeaders200, 200, 'team-test'); +assert('200: type = teams-direct', parsed200.type === 'teams-direct'); +assert('200: status = allowed', parsed200.status === 'allowed'); +assert('200: utilization_5h is number', typeof parsed200.utilization_5h === 'number'); +assert('200: utilization_5h = 0.32', parsed200.utilization_5h === 0.32); +assert('200: utilization_7d = 0.45', parsed200.utilization_7d === 0.45); +assert('200: representative_claim present', parsed200.representative_claim === 'five_hour'); +assert('200: reset_timestamp is number', typeof parsed200.reset_timestamp === 'number'); +assert('200: reset_in_seconds is number', typeof parsed200.reset_in_seconds === 'number'); +assert('200: organization_id present', parsed200.organization_id === '1d7653ad-11d9-4029-ba1a-a2dd4cd0b2f3'); +assert('200: severity = ok', parsed200.severity === 'ok'); + +// Simulate 429 with rejected status +const rejected429headers = new Map([ + ['anthropic-ratelimit-unified-status', 'rejected'], + ['anthropic-ratelimit-unified-5h-status', 'allowed'], + ['anthropic-ratelimit-unified-5h-utilization', '0.0'], + ['anthropic-ratelimit-unified-5h-reset', '1775322000'], + ['anthropic-ratelimit-unified-7d-status', 'rejected'], + ['anthropic-ratelimit-unified-7d-utilization', '1.0'], + ['anthropic-ratelimit-unified-7d-surpassed-threshold', '1.0'], + ['anthropic-ratelimit-unified-7d-reset', '1775404800'], + ['anthropic-ratelimit-unified-representative-claim', 'seven_day'], + ['anthropic-ratelimit-unified-fallback-percentage', '0.5'], + ['anthropic-ratelimit-unified-reset', '1775404800'], + ['anthropic-ratelimit-unified-overage-status', 'rejected'], + ['anthropic-ratelimit-unified-overage-disabled-reason', 'org_level_disabled'], + ['anthropic-organization-id', '1d7653ad-11d9-4029-ba1a-a2dd4cd0b2f3'], + ['retry-after', '83690'], +]); +const fakeHeaders429 = { get: (k) => rejected429headers.get(k.toLowerCase()) || null }; +const parsed429 = parseTeamsHeaders(fakeHeaders429, 429, 'team-vigilio'); +assert('429: status = rejected', parsed429.status === 'rejected'); +assert('429: utilization_7d = 1.0', parsed429.utilization_7d === 1.0); +assert('429: severity = critical', parsed429.severity === 'critical'); +assert('429: retry_after_seconds = 83690', parsed429.retry_after_seconds === 83690); + +// 401 invalid key +const { parseTeamsHeaders: ph } = await import('./providers/anthropic-teams.js'); +const parsed401 = ph({ get: () => null }, 401, 'team-ludo'); +assert('401: status = invalid_key', parsed401.status === 'invalid_key'); +assert('401: utilization_5h = null', parsed401.utilization_5h === null); +assert('401: utilization_7d = null', parsed401.utilization_7d === null); +assert('401: severity = unknown', parsed401.severity === 'unknown'); + +// ── 6. Shelley header parser ───────────────────────────────────────────────── +console.log('\n── 6. Shelley header parser ────────────────────────────────────'); +const { parseShelleyHeaders } = await import('./providers/shelley-proxy.js'); + +const shelleyHeaderMap = new Map([ + ['anthropic-ratelimit-tokens-limit', '4800000'], + ['anthropic-ratelimit-tokens-remaining', '4750000'], + ['anthropic-ratelimit-tokens-reset', '2026-04-04T13:00:00Z'], + ['anthropic-ratelimit-requests-limit', '20000'], + ['anthropic-ratelimit-requests-remaining', '19999'], + ['anthropic-ratelimit-requests-reset', '2026-04-04T16:45:09Z'], + ['exedev-gateway-cost', '0.000013'], +]); +const fakeShelleyHeaders = { get: (k) => shelleyHeaderMap.get(k.toLowerCase()) || null }; +const parsedShelley = parseShelleyHeaders(fakeShelleyHeaders, 200); +assert('shelley: type = shelley-proxy', parsedShelley.type === 'shelley-proxy'); +assert('shelley: status = ok', parsedShelley.status === 'ok'); +assert('shelley: tokens_limit = 4800000', parsedShelley.tokens_limit === 4800000); +assert('shelley: tokens_remaining = 4750000', parsedShelley.tokens_remaining === 4750000); +assert('shelley: tokens_reset present', typeof parsedShelley.tokens_reset === 'string'); +assert('shelley: requests_limit = 20000', parsedShelley.requests_limit === 20000); +assert('shelley: requests_remaining = 19999', parsedShelley.requests_remaining === 19999); +assert('shelley: cost_per_call_usd = 0.000013', parsedShelley.cost_per_call_usd === 0.000013); +assert('shelley: severity = ok', parsedShelley.severity === 'ok'); + +// ── 7. api-ateam provider ──────────────────────────────────────────────────── +console.log('\n── 7. api-ateam (no billing data) ──────────────────────────────'); +const { getApiAteamStatus } = await import('./providers/anthropic-api.js'); +const apiStatus = getApiAteamStatus(); +assert('api-ateam: type = api-direct', apiStatus.type === 'api-direct'); +assert('api-ateam: status = no_billing_data', apiStatus.status === 'no_billing_data'); +assert('api-ateam: message is string', typeof apiStatus.message === 'string' && apiStatus.message.length > 0); +assert('api-ateam: severity = unknown', apiStatus.severity === 'unknown'); + +// ── 8. Logger ──────────────────────────────────────────────────────────────── +console.log('\n── 8. Logger ───────────────────────────────────────────────────'); +const { logRun } = await import('./logger.js'); +const testData = { test: true, value: 42, providers: {} }; +logRun(testData); +const today = new Date().toISOString().slice(0, 10); +const logFile = join(homedir(), '.logs', 'token-monitor', `${today}.jsonl`); +assert('log file created', existsSync(logFile)); +const lastLine = readFileSync(logFile, 'utf-8').trim().split('\n').pop(); +const logEntry = JSON.parse(lastLine); +assert('log entry has ts', typeof logEntry.ts === 'string'); +assert('log entry has test data', logEntry.test === true && logEntry.value === 42); + +// ── 9. Report generator ────────────────────────────────────────────────────── +console.log('\n── 9. Report generator ─────────────────────────────────────────'); +const { generateReport } = await import('./report.js'); +const sampleResult = { + timestamp: '2026-04-04T12:00:00Z', + providers: { + 'team-vigilio': { + type: 'teams-direct', status: 'rejected', + utilization_5h: 0.0, utilization_7d: 1.0, + representative_claim: 'seven_day', + reset_timestamp: Math.floor(Date.now() / 1000) + 82800, + reset_in_seconds: 82800, + severity: 'critical', + }, + 'team-molto': { + type: 'teams-direct', status: 'allowed', + utilization_5h: 0.32, utilization_7d: 0.45, + representative_claim: 'five_hour', + reset_timestamp: Math.floor(Date.now() / 1000) + 3600, + reset_in_seconds: 3600, + severity: 'ok', + }, + 'team-ludo': { + type: 'teams-direct', status: 'invalid_key', + utilization_5h: null, utilization_7d: null, + severity: 'unknown', + }, + 'shelley-proxy': { + type: 'shelley-proxy', status: 'ok', + tokens_remaining: 45000, tokens_limit: 50000, + requests_remaining: 48, requests_limit: 50, + tokens_reset: '2026-04-04T13:00:00Z', + cost_per_call_usd: 0.000013, + severity: 'ok', + }, + 'api-ateam': { + type: 'api-direct', status: 'no_billing_data', + message: 'Anthropic API does not expose billing/quota via REST', + severity: 'unknown', + }, + }, +}; +const report = generateReport(sampleResult); +assert('report is string', typeof report === 'string'); +assert('report contains Token Monitor header', report.includes('Token Monitor')); +assert('report contains team-vigilio', report.includes('team-vigilio')); +assert('report contains CRITICAL', report.includes('CRITICAL')); +assert('report contains team-molto', report.includes('team-molto')); +assert('report contains shelley-proxy', report.includes('shelley-proxy')); +assert('report contains api-ateam', report.includes('api-ateam')); +assert('report contains Overall summary', report.toLowerCase().includes('overall')); + +// ── 10. CLI: --summary flag ────────────────────────────────────────────────── +console.log('\n── 10. CLI flags (--summary, --json) ───────────────────────────'); +// --summary: exit 0, stdout contains provider names +const summaryResult = runSafe('node monitor.js --summary'); +assert('--summary exits 0', summaryResult.code === 0, `exit code: ${summaryResult.code}\n${summaryResult.stderr || ''}`); +assert('--summary output contains Token Monitor', summaryResult.stdout.includes('Token Monitor')); +assert('--summary output contains provider names', summaryResult.stdout.includes('team-') || summaryResult.stdout.includes('shelley')); + +// --json: exit 0, valid JSON, correct schema +const jsonResult = runSafe('node monitor.js --json'); +assert('--json exits 0', jsonResult.code === 0, `exit code: ${jsonResult.code}\n${jsonResult.stderr || ''}`); +let jsonData; +try { + jsonData = JSON.parse(jsonResult.stdout); + assert('--json outputs valid JSON', true); +} catch (e) { + assert('--json outputs valid JSON', false, e.message); +} +if (jsonData) { + assert('JSON has timestamp', typeof jsonData.timestamp === 'string'); + assert('JSON has providers object', typeof jsonData.providers === 'object'); + const pnames = Object.keys(jsonData.providers); + assert('JSON providers includes team-vigilio', pnames.includes('team-vigilio')); + assert('JSON providers includes shelley-proxy', pnames.includes('shelley-proxy')); + assert('JSON providers includes api-ateam', pnames.includes('api-ateam')); + + // Teams-direct entries must have required fields + for (const [name, p] of Object.entries(jsonData.providers)) { + if (p.type === 'teams-direct' && p.status !== 'invalid_key') { + assert(`${name}: has utilization_5h`, typeof p.utilization_5h === 'number'); + assert(`${name}: has utilization_7d`, typeof p.utilization_7d === 'number'); + assert(`${name}: has severity`, typeof p.severity === 'string'); + assert(`${name}: has reset_in_seconds`, typeof p.reset_in_seconds === 'number'); + } + if (p.type === 'shelley-proxy') { + assert('shelley-proxy: has cost_per_call_usd', typeof p.cost_per_call_usd === 'number'); + assert('shelley-proxy: has tokens_remaining', typeof p.tokens_remaining === 'number'); + assert('shelley-proxy: has tokens_limit', typeof p.tokens_limit === 'number'); + } + if (p.type === 'api-direct') { + assert('api-ateam: status = no_billing_data', p.status === 'no_billing_data'); + } + } +} + +// Default run: exit 0, log file updated +const defaultResult = runSafe('node monitor.js'); +assert('default run exits 0', defaultResult.code === 0, `exit code: ${defaultResult.code}\n${defaultResult.stderr || ''}`); + +// Log file should exist and have content +const logFileAfter = join(homedir(), '.logs', 'token-monitor', `${today}.jsonl`); +assert('log file exists after default run', existsSync(logFileAfter)); +const logLines = readFileSync(logFileAfter, 'utf-8').trim().split('\n'); +assert('log file has at least 2 entries', logLines.length >= 2); + +// ── Results ────────────────────────────────────────────────────────────────── +console.log('\n' + '═'.repeat(50)); +console.log(`Tests: ${passed + failed} | Passed: ${passed} | Failed: ${failed}`); +if (failures.length > 0) { + console.log('\nFailed:'); + for (const f of failures) { + console.log(` ✗ ${f.label}${f.detail ? ' — ' + f.detail : ''}`); + } + process.exit(1); +} else { + console.log('\nAll tests pass. ✓'); + process.exit(0); +}