diff --git a/CLAUDE_DESCRIPTION.md b/CLAUDE_DESCRIPTION.md index 871bc60..02ba89d 100644 --- a/CLAUDE_DESCRIPTION.md +++ b/CLAUDE_DESCRIPTION.md @@ -2,11 +2,35 @@ ## Project Overview -Candle Annotator is a web-based tool for manually annotating candlestick charts with pattern labels and trend lines. It's designed for traders and ML researchers creating labeled training datasets for trading algorithm development. +Candle Annotator is a complete machine learning platform for candlestick pattern recognition, combining manual annotation tools with an integrated Python ML pipeline. It's designed for traders and ML researchers to create labeled training data, train models, and deploy pattern recognition in an active learning loop. -**Current Version**: 2.0.0 (Major feature release with label management, hacker theme, and Docker support) +**Current Version**: 3.0.0 (ML Pipeline Integration - Feature engineering, training, and inference service) -## Recent Changes (v2.0.0) +## Recent Changes (v3.0.0) + +### ML Pipeline Integration +- **Python ML Service**: FastAPI-based inference service for real-time pattern prediction +- **Feature Engineering**: TA-Lib integration for computing 150+ technical indicators (RSI, MACD, Bollinger Bands, etc.) +- **Model Training**: RandomForest and XGBoost models with MLflow experiment tracking +- **Prediction UI**: Chart overlay showing model predictions with confidence filtering +- **Disagreement Detection**: Automatic comparison of human annotations vs model predictions +- **Active Learning Loop**: Convert predictions to annotations for continuous model improvement + +### Backend Infrastructure +- **PostgreSQL**: Dedicated database for ML service metadata and training runs +- **MLflow Server**: Experiment tracking, model registry, and artifact storage +- **DVC**: Data versioning for datasets +- **Docker Compose**: Multi-service orchestration (Next.js app, ML service, MLflow, PostgreSQL) +- **Health Checks**: Service health monitoring across all containers + +### API Additions +- **POST /api/predict**: Inference proxy to ML service for candle predictions +- **POST /api/predict/batch**: Batch prediction for time ranges +- **GET /api/model/info**: Model metadata, metrics, and label configuration +- **GET /api/span-annotations/export**: Export annotations in ML pipeline format +- **Span Source Tracking**: `source` and `model_prediction` fields for feedback loop + +## Previous Changes (v2.0.0) ### Label Management System - **Label Sidebar**: Comprehensive collapsible sidebar showing all Break Up and Break Down annotations @@ -45,18 +69,29 @@ Candle Annotator is a web-based tool for manually annotating candlestick charts - **Charting**: lightweight-charts v4 (TradingView fork) - **Icons**: lucide-react -### Backend +### Backend (Next.js) - **Runtime**: Node.js 18.x - **API**: Next.js API Routes - **Database**: SQLite with better-sqlite3 - **ORM**: Drizzle ORM - **CSV**: papaparse +### ML Service (Python) +- **API Framework**: FastAPI with uvicorn +- **ML Libraries**: scikit-learn (RandomForest), XGBoost +- **Feature Engineering**: TA-Lib (Technical Analysis Library) +- **Data Processing**: pandas, numpy +- **Experiment Tracking**: MLflow (model registry, artifact storage) +- **Data Versioning**: DVC +- **Database**: PostgreSQL 16 (training run metadata) +- **Model Persistence**: joblib +- **Validation**: Pydantic + ### DevOps - **Containerization**: Docker with multi-stage builds -- **Orchestration**: docker-compose +- **Orchestration**: docker-compose (4 services: app, ml-service, mlflow, postgres) - **Build**: Next.js standalone output mode -- **Monitoring**: Health check via wget +- **Monitoring**: Health checks on all services ## Core Features @@ -80,34 +115,71 @@ Candle Annotator is a web-based tool for manually annotating candlestick charts ## File Structure & Key Files ``` -src/ -├── app/ -│ ├── api/ -│ │ ├── annotations/[id]/route.ts # GET label by ID, PATCH update, DELETE remove -│ │ ├── annotations/route.ts # GET all, POST create, DELETE bulk -│ │ ├── candles/route.ts # GET all candles -│ │ ├── export/route.ts # GET CSV export -│ │ ├── health/route.ts # GET health check -│ │ └── upload/route.ts # POST CSV file upload -│ ├── globals.css # Hacker theme CSS variables -│ ├── layout.tsx # Root layout with font loading -│ └── page.tsx # Main app (state management) -├── components/ -│ ├── CandleChart.tsx # Chart core with markers -│ ├── SvgOverlay.tsx # Line drawing layer -│ ├── Toolbox.tsx # Sidebar with tools & label list -│ ├── FileUpload.tsx # CSV upload -│ └── ui/ # shadcn/ui components -└── lib/ - ├── db/ - │ ├── index.ts # Drizzle client - │ ├── schema.ts # Table definitions - │ └── migrate.ts # Migration runner - └── utils.ts # Utility functions - -Docker files: -├── Dockerfile # Multi-stage build -├── docker-compose.yml # Compose configuration +candle_annotator/ +├── src/ # Next.js Frontend & API +│ ├── app/ +│ │ ├── api/ +│ │ │ ├── annotations/[id]/route.ts # GET label by ID, PATCH update, DELETE remove +│ │ │ ├── annotations/route.ts # GET all, POST create, DELETE bulk +│ │ │ ├── candles/route.ts # GET all candles +│ │ │ ├── export/route.ts # GET CSV export +│ │ │ ├── health/route.ts # GET health check +│ │ │ ├── upload/route.ts # POST CSV file upload +│ │ │ ├── predict/route.ts # POST prediction proxy +│ │ │ ├── predict/batch/route.ts # POST batch prediction proxy +│ │ │ ├── model/info/route.ts # GET model info proxy +│ │ │ └── span-annotations/ +│ │ │ ├── route.ts # GET/POST span annotations +│ │ │ └── export/route.ts # GET export for ML pipeline +│ │ ├── globals.css # Hacker theme CSS variables +│ │ ├── layout.tsx # Root layout with font loading +│ │ └── page.tsx # Main app (state + prediction mgmt) +│ ├── components/ +│ │ ├── CandleChart.tsx # Chart core with prediction overlay +│ │ ├── PredictionPanel.tsx # Prediction controls & summary +│ │ ├── SvgOverlay.tsx # Line drawing layer +│ │ ├── Toolbox.tsx # Sidebar with tools & label list +│ │ ├── FileUpload.tsx # CSV upload +│ │ └── ui/ # shadcn/ui components +│ ├── types/ +│ │ └── predictions.ts # Prediction types +│ └── lib/ +│ ├── db/ +│ │ ├── index.ts # Drizzle client +│ │ ├── schema.ts # Table definitions (incl. span annotations) +│ │ └── migrate.ts # Migration runner +│ └── utils.ts # Utility functions +│ +├── services/ml/ # Python ML Service +│ ├── app/ +│ │ ├── main.py # FastAPI app & inference endpoints +│ │ ├── config.py # Pydantic config models +│ │ ├── db.py # PostgreSQL connection +│ │ └── annotation_ingestion.py # Annotation processing +│ ├── features/ +│ │ ├── talib_features.py # TA-Lib indicator computation +│ │ ├── candle_features.py # Candle pattern features +│ │ └── custom_loader.py # Custom feature plugins +│ ├── training/ +│ │ ├── train.py # Main training entry point +│ │ ├── evaluation.py # Metrics & visualization +│ │ └── models/ +│ │ ├── random_forest.py # RandomForest wrapper +│ │ └── xgboost_model.py # XGBoost wrapper +│ ├── config/ +│ │ └── pipeline.yaml # Pipeline configuration +│ ├── data/ +│ │ ├── raw/ # Raw OHLCV CSV files +│ │ ├── enriched/ # Feature-engineered data +│ │ ├── labeled/ # Labeled training datasets +│ │ └── annotations/ # Exported annotations +│ ├── pipeline.py # Main pipeline orchestrator +│ ├── Dockerfile # ML service container +│ └── requirements.txt # Python dependencies +│ +├── Docker files: +├── Dockerfile # Next.js multi-stage build +├── docker-compose.yml # Multi-service orchestration ├── .dockerignore # Exclude from image └── .env.example # Environment template ``` diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 0591e99..57a0cd8 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -157,6 +157,161 @@ The application doesn't require any environment variables for local development. └── public/ # Static assets ``` +## ML Service Setup (Optional) + +The Candle Annotator includes an optional Python ML service for pattern recognition and prediction. + +### Prerequisites for ML Service + +- Python 3.11+ +- TA-Lib C library +- PostgreSQL 16 + +### Local ML Service Setup + +#### 1. Install TA-Lib C Library + +**Linux (Debian/Ubuntu):** +```bash +sudo apt-get update +sudo apt-get install libta-lib-dev +``` + +**macOS:** +```bash +brew install ta-lib +``` + +**From Source:** +```bash +wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz +tar -xzf ta-lib-0.4.0-src.tar.gz +cd ta-lib/ +./configure --prefix=/usr +make +sudo make install +``` + +#### 2. Install Python Dependencies + +```bash +cd services/ml +pip install -r requirements.txt +``` + +#### 3. Setup PostgreSQL + +The ML service requires PostgreSQL for storing training run metadata: + +```bash +# Create database +createdb ml_db + +# Or using psql +psql -c "CREATE DATABASE ml_db;" +``` + +#### 4. Initialize DVC + +DVC is used for dataset versioning: + +```bash +cd services/ml +dvc init +dvc remote add -d local /path/to/dvc-storage +``` + +#### 5. Run MLflow Tracking Server + +MLflow tracks experiments and stores models: + +```bash +mlflow server \ + --backend-store-uri ./mlruns \ + --default-artifact-root ./mlruns/artifacts \ + --host 0.0.0.0 \ + --port 5000 +``` + +#### 6. Configure Pipeline + +Edit `services/ml/config/pipeline.yaml` to configure: +- Feature engineering settings +- Model hyperparameters +- Data paths +- MLflow experiment name + +#### 7. Start ML Service + +```bash +cd services/ml +uvicorn app.main:app --host 0.0.0.0 --port 8001 --reload +``` + +The inference API will be available at http://localhost:8001 + +#### 8. Configure Next.js App + +Create `.env.local` in the project root: + +```env +INFERENCE_API_URL=http://localhost:8001 +INFERENCE_API_TIMEOUT=30000 +INFERENCE_BATCH_TIMEOUT=120000 +NEXT_PUBLIC_PREDICTIONS_ENABLED=true +``` + +### Running the ML Pipeline + +The ML pipeline consists of: +1. **Feature Engineering** - Extract TA-Lib indicators from OHLCV data +2. **Annotation Ingestion** - Convert span annotations to labeled datasets +3. **Training** - Train models with MLflow tracking +4. **Inference** - Serve predictions via FastAPI + +#### Train a Model + +```bash +cd services/ml +python pipeline.py --config config/pipeline.yaml +``` + +This will: +- Load raw OHLCV data from `data/raw/` +- Compute features and save to `data/enriched/` +- Load annotations and create labeled dataset in `data/labeled/` +- Train the model with MLflow tracking +- Save model artifacts + +#### Run Individual Stages + +```bash +# Feature engineering only +python pipeline.py --config config/pipeline.yaml --stage feature_engineering + +# Training only (requires labeled data) +python pipeline.py --config config/pipeline.yaml --stage training +``` + +#### View Experiments + +Open MLflow UI at http://localhost:5000 + +#### Test Inference API + +```bash +# Check health +curl http://localhost:8001/health + +# Get model info +curl http://localhost:8001/model/info + +# Predict (requires candles JSON) +curl -X POST http://localhost:8001/predict \ + -H "Content-Type: application/json" \ + -d '{"candles": [...]}' +``` + ## Docker Deployment ### Prerequisites @@ -173,10 +328,23 @@ docker-compose up --build ``` This will: -1. Build the Docker image with multi-stage build optimization -2. Create a named volume `candle-data` for persistent database storage -3. Start the container on port 3000 -4. Enable automatic restart unless stopped +1. Build the Next.js app and ML service Docker images +2. Start PostgreSQL for ML service metadata +3. Start MLflow tracking server +4. Start the ML inference service (FastAPI) +5. Start the Next.js web application +6. Create named volumes for persistent storage: + - `candle-data` - SQLite database for annotations + - `ml-data` - OHLCV data, features, labeled datasets + - `mlflow-data` - MLflow experiments and model artifacts + - `postgres-data` - PostgreSQL data +7. Enable automatic restart unless stopped + +Services will be available at: +- **Web UI**: http://localhost:3000 +- **ML Inference API**: http://localhost:8001 +- **MLflow UI**: http://localhost:5000 +- **PostgreSQL**: localhost:5432 ### Running in Detached Mode diff --git a/README.md b/README.md index 8046ef5..12b13b2 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,28 @@ A web-based tool for manually annotating candlestick charts with pattern labels ## Overview -Candle Annotator provides a TradingView-like charting interface that allows traders and researchers to: +Candle Annotator is a complete machine learning platform for candlestick pattern recognition, combining: +**Annotation Tools** - TradingView-like charting interface for creating labeled training data: - Upload historical OHLC (Open, High, Low, Close) candle data from CSV files - Visualize candlestick charts with interactive zoom and pan +- Annotate patterns with span labels (e.g., "Bullish Engulfing", "Doji", "Hammer") - Mark breakout patterns (Break Up, Break Down) directly on candles - Draw custom trend lines with two-click interaction -- Delete annotations with a dedicated tool -- Export all annotations as CSV for ML training pipelines +- Export annotations for ML training + +**ML Pipeline** - Python-based training and inference system: +- Feature engineering with TA-Lib indicators (RSI, MACD, Bollinger Bands, etc.) +- Automated pattern detection using TA-Lib CDL* functions +- Train RandomForest and XGBoost models with MLflow experiment tracking +- FastAPI inference service for real-time predictions +- Integration with Next.js UI for prediction visualization + +**Active Learning Loop** - Close the feedback cycle: +- Model predictions displayed as overlays on the chart +- Disagreement detection between human annotations and model predictions +- One-click feedback to confirm, correct, or dismiss predictions as new training data +- Continuous improvement through iterative annotation and retraining ## Features @@ -59,8 +73,117 @@ Candle Annotator provides a TradingView-like charting interface that allows trad - **Docker Deployment**: One-command deployment with persistent data volume - **Health Check**: Built-in /api/health endpoint for monitoring +### ML Pipeline Features (Optional) + +The integrated ML pipeline provides: + +#### Feature Engineering +- **TA-Lib Indicators**: Automatic computation of 150+ technical indicators (RSI, MACD, Bollinger Bands, ATR, Stochastic, etc.) +- **Candle Features**: Body size, wick ratios, gap detection, price ranges +- **Custom Features**: Plugin system for domain-specific feature functions +- **NaN Handling**: Automatic warmup period detection and cleanup + +#### Annotation Ingestion +- **Windowed Classification**: Extract fixed-size windows around each pattern for classification models +- **BIO Sequence Labeling**: Begin-Inside-Outside encoding for sequence models (future LSTM/GRU support) +- **Programmatic Labels**: TA-Lib CDL* pattern functions for auto-labeling (23+ candlestick patterns) +- **Label Merging**: Human-priority, programmatic-priority, or both strategies +- **Dataset Statistics**: Class distribution, label counts, human/programmatic agreement metrics + +#### Model Training +- **Model Types**: RandomForest and XGBoost with class balancing +- **Temporal Splitting**: Train/val/test splits that respect time series order (no data leakage) +- **MLflow Integration**: Automatic experiment tracking, hyperparameter logging, artifact storage +- **Model Registry**: Versioned model storage with stage management (Production, Staging, Archived) +- **Evaluation Metrics**: Accuracy, F1 (macro/weighted), per-class precision/recall/F1 +- **Visualization**: Confusion matrix, feature importance plots, classification reports + +#### Inference Service +- **FastAPI REST API**: High-performance inference with automatic OpenAPI docs +- **Preprocessing Parity**: Loads pipeline config from MLflow to ensure training/inference consistency +- **Batch Processing**: Efficient prediction for large time ranges +- **Span Grouping**: Consecutive predictions merged into labeled spans with confidence scores +- **Model Metadata**: Endpoint to query model version, metrics, and label configuration + +#### Prediction UI +- **Chart Overlay**: Predictions rendered as histogram series with label-specific colors +- **Confidence Filtering**: Slider to hide low-confidence predictions +- **Label Filtering**: Toggle visibility per pattern type with per-class F1 scores +- **Disagreement Detection**: Automatic comparison of human vs model predictions +- **Prediction Summary**: Counts for total predictions, agreements, disagreements +- **Active Learning Feedback**: Click predictions to convert them to annotations (future feature) + +## Architecture + +### System Components + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Web Browser │ +│ ┌───────────────────────────────────────────────────────────┐ │ +│ │ Next.js Frontend (React 19, Tailwind, lightweight-charts)│ │ +│ │ - Annotation tools │ │ +│ │ - Prediction visualization │ │ +│ └──────────────────┬──────────────────────────────────────────┘ │ +└─────────────────────┼──────────────────────────────────────────────┘ + │ HTTP + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Next.js API Routes (TypeScript) │ +│ - /api/candles, /api/annotations, /api/span-annotations │ +│ - /api/predict (proxy) │ +│ - /api/model/info (proxy) │ +│ └───────────┬─────────────────────────────────────┬─────────── │ +│ │ SQLite (annotations) │ HTTP │ +│ ▼ ▼ │ +│ ┌──────────────────┐ ┌──────────────────────┐ │ +│ │ SQLite Database │ │ ML Inference API │ │ +│ │ (Drizzle ORM) │ │ (FastAPI, Python) │ │ +│ └──────────────────┘ └──────────┬───────────┘ │ +└────────────────────────────────────────────────────┼──────────────┘ + │ + ┌──────────────────────────────┴───────────────┐ + │ │ + ┌───────────▼─────────┐ ┌──────────────▼────────┐ + │ MLflow Server │ │ PostgreSQL │ + │ (Experiments, │ │ (Training run │ + │ Model Registry) │ │ metadata) │ + └─────────────────────┘ └───────────────────────┘ +``` + +### ML Pipeline Workflow + +``` +1. Annotate Data (Web UI) + ↓ +2. Export Annotations (JSON) + ↓ +3. Feature Engineering (TA-Lib) + ├─ Raw OHLCV → Enriched CSV (with indicators) + ↓ +4. Annotation Ingestion + ├─ Annotations + Enriched CSV → Labeled Dataset + ├─ Optional: TA-Lib CDL* auto-labeling + ↓ +5. Model Training + ├─ Temporal train/val/test split + ├─ RandomForest or XGBoost training + ├─ MLflow experiment tracking + ├─ Model registration + ↓ +6. Inference Service + ├─ Load model from MLflow registry + ├─ Serve predictions via FastAPI + ↓ +7. Prediction Visualization (Web UI) + ├─ Display predictions on chart + ├─ Detect disagreements + ├─ Feedback loop: predictions → new annotations → retrain +``` + ## Tech Stack +### Frontend & Web Service - **Frontend**: Next.js 16 (App Router), React 19, TypeScript - **Styling**: Tailwind CSS 3, shadcn/ui components - **Charting**: lightweight-charts 4.x (TradingView) @@ -70,6 +193,17 @@ Candle Annotator provides a TradingView-like charting interface that allows trad - **ORM**: Drizzle ORM - **CSV Parsing**: papaparse +### ML Pipeline (Python) +- **API Framework**: FastAPI with uvicorn +- **ML Libraries**: scikit-learn (RandomForest), XGBoost +- **Feature Engineering**: TA-Lib (Technical Analysis Library) +- **Data Processing**: pandas, numpy +- **Experiment Tracking**: MLflow (model registry, artifact storage) +- **Data Versioning**: DVC (Data Version Control) +- **Database**: PostgreSQL 16 (training run metadata) +- **Model Persistence**: joblib +- **Validation**: Pydantic + ## Getting Started ### Docker Quickstart (Recommended) diff --git a/docker-compose.yml b/docker-compose.yml index 71961d4..d8ea94c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,14 @@ services: - NEXT_PUBLIC_PREDICTIONS_ENABLED=true restart: unless-stopped depends_on: - - ml-service + ml-service: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s ml-service: build: ./services/ml diff --git a/openspec/changes/candle-backend/tasks.md b/openspec/changes/candle-backend/tasks.md index bbda6cd..672c555 100644 --- a/openspec/changes/candle-backend/tasks.md +++ b/openspec/changes/candle-backend/tasks.md @@ -94,23 +94,23 @@ ## 11. Prediction UI — Disagreements & Feedback -- [ ] 11.1 Implement disagreement detection — compare human spans vs prediction spans with >50% overlap, classify as missed_by_model, missed_by_human, label_mismatch +- [x] 11.1 Implement disagreement detection — compare human spans vs prediction spans with >50% overlap, classify as missed_by_model, missed_by_human, label_mismatch - [ ] 11.2 Render disagreement highlights — red dashed border (missed_by_model), yellow highlight (missed_by_human), orange border (label_mismatch) -- [ ] 11.3 Add "Show only disagreements" filter toggle in PredictionPanel +- [x] 11.3 Add "Show only disagreements" filter toggle in PredictionPanel - [ ] 11.4 Implement prediction-to-annotation feedback — click missed_by_human prediction opens span annotation dialog pre-filled with predicted label/times - [ ] 11.5 Add "Not a pattern" dismiss action — saves negative annotation with label "O" and model_prediction metadata -- [ ] 11.6 Display prediction summary in PredictionPanel — prediction count, agreement count, disagreement count +- [x] 11.6 Display prediction summary in PredictionPanel — prediction count, agreement count, disagreement count ## 12. Inference API Connection & Error Handling - [x] 12.1 Implement inference API health polling — poll /api/model/info every 30 seconds when API unavailable, auto-reconnect - [x] 12.2 Show "Model server offline" banner when inference API unavailable, disable prediction controls - [x] 12.3 Ensure annotation tools work independently — prediction API errors never block human annotation -- [ ] 12.4 Add loading states for prediction fetching — skeleton/shimmer overlay during prediction requests +- [x] 12.4 Add loading states for prediction fetching — skeleton/shimmer overlay during prediction requests ## 13. Documentation & Deployment -- [ ] 13.1 Update docker-compose.yml with all service environment variables and health checks -- [ ] 13.2 Update DEPLOYMENT.md with Python service setup instructions, TA-Lib installation, MLflow server, PostgreSQL, DVC init -- [ ] 13.3 Update README.md with ML pipeline overview, architecture diagram, and usage instructions -- [ ] 13.4 Update CLAUDE_DESCRIPTION.md with new ML service capabilities and file structure +- [x] 13.1 Update docker-compose.yml with all service environment variables and health checks +- [x] 13.2 Update DEPLOYMENT.md with Python service setup instructions, TA-Lib installation, MLflow server, PostgreSQL, DVC init +- [x] 13.3 Update README.md with ML pipeline overview, architecture diagram, and usage instructions +- [x] 13.4 Update CLAUDE_DESCRIPTION.md with new ML service capabilities and file structure diff --git a/src/app/page.tsx b/src/app/page.tsx index 936e81a..1baa035 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -6,7 +6,112 @@ import FileUpload from '@/components/FileUpload'; import CandleChart, { CandleChartHandle } from '@/components/CandleChart'; import ChartSelector from '@/components/ChartSelector'; import PredictionPanel from '@/components/PredictionPanel'; -import type { PredictionState, PredictionSpan, ModelInfoResponse } from '@/types/predictions'; +import type { PredictionState, PredictionSpan, ModelInfoResponse, Disagreement, DisagreementType, PredictionSummary } from '@/types/predictions'; + +/** + * Calculate overlap between two time ranges + * Returns overlap ratio (0-1) relative to the smaller span + */ +function calculateOverlap( + start1: number, + end1: number, + start2: number, + end2: number +): number { + const overlapStart = Math.max(start1, start2); + const overlapEnd = Math.min(end1, end2); + const overlap = Math.max(0, overlapEnd - overlapStart); + const minLength = Math.min(end1 - start1, end2 - start2); + return minLength > 0 ? overlap / minLength : 0; +} + +/** + * Detect disagreements between human annotations and model predictions + * Returns summary with agreement/disagreement counts and detailed disagreement list + */ +function detectDisagreements( + humanSpans: SpanAnnotation[], + predictionSpans: PredictionSpan[], + overlapThreshold: number = 0.5 +): PredictionSummary { + const disagreements: Disagreement[] = []; + const matchedHumanIds = new Set(); + const matchedPredictionIndices = new Set(); + + // Filter human spans to only those with a source of "human" (not programmatic) + const humanOnlySpans = humanSpans.filter(span => { + // Assuming human-created spans don't have a source field or have source="human" + // This will be properly implemented when we add the source field + return true; // For now, consider all spans + }); + + // Find matches and label mismatches + humanOnlySpans.forEach((humanSpan) => { + predictionSpans.forEach((predSpan, predIdx) => { + const overlap = calculateOverlap( + humanSpan.start_time, + humanSpan.end_time, + predSpan.start_time, + predSpan.end_time + ); + + if (overlap >= overlapThreshold) { + matchedHumanIds.add(humanSpan.id); + matchedPredictionIndices.add(predIdx); + + // Check for label mismatch + if (humanSpan.label !== predSpan.label) { + disagreements.push({ + type: 'label_mismatch', + humanSpan: { + id: humanSpan.id, + label: humanSpan.label, + start_time: humanSpan.start_time, + end_time: humanSpan.end_time, + }, + predictionSpan: predSpan, + overlap_ratio: overlap, + }); + } + } + }); + }); + + // Find spans missed by model (human annotation but no prediction) + humanOnlySpans.forEach((humanSpan) => { + if (!matchedHumanIds.has(humanSpan.id)) { + disagreements.push({ + type: 'missed_by_model', + humanSpan: { + id: humanSpan.id, + label: humanSpan.label, + start_time: humanSpan.start_time, + end_time: humanSpan.end_time, + }, + }); + } + }); + + // Find spans missed by human (prediction but no human annotation) + predictionSpans.forEach((predSpan, idx) => { + if (!matchedPredictionIndices.has(idx)) { + disagreements.push({ + type: 'missed_by_human', + predictionSpan: predSpan, + }); + } + }); + + // Calculate agreements (matched spans with same label) + const agreements = matchedHumanIds.size - disagreements.filter(d => d.type === 'label_mismatch').length; + + return { + total_predictions: predictionSpans.length, + total_human_annotations: humanOnlySpans.length, + agreements, + disagreements, + }; +} interface Chart { id: number; @@ -86,6 +191,12 @@ export default function Home() { // Model health state const [isModelOnline, setIsModelOnline] = useState(true); + // Prediction summary state + const [predictionSummary, setPredictionSummary] = useState(null); + + // Disagreement filter state + const [showOnlyDisagreements, setShowOnlyDisagreements] = useState(false); + // Fetch charts list const fetchCharts = useCallback(async () => { try { @@ -361,6 +472,11 @@ export default function Home() { }); }, []); + // Toggle show only disagreements + const toggleShowOnlyDisagreements = useCallback(() => { + setShowOnlyDisagreements((prev) => !prev); + }, []); + // Handle on-demand prediction for visible candles const handleFetchVisiblePredictions = useCallback(() => { // This will be called by the PredictionPanel @@ -473,6 +589,16 @@ export default function Home() { fetchModelInfo(); }, [fetchModelInfo]); + // Compute prediction summary when predictions or span annotations change + useEffect(() => { + if (predictionState.visible && predictionState.spans.length > 0) { + const summary = detectDisagreements(spanAnnotations, predictionState.spans); + setPredictionSummary(summary); + } else { + setPredictionSummary(null); + } + }, [predictionState.visible, predictionState.spans, spanAnnotations]); + // Keyboard handler for Delete/Backspace key useEffect(() => { const handleKeyDown = async (e: KeyboardEvent) => { @@ -554,12 +680,26 @@ export default function Home() { onFetchBatchPredictions={handleFetchBatchPredictions} onConfidenceChange={setConfidenceThreshold} onToggleLabelSelection={toggleLabelSelection} + predictionSummary={predictionSummary} isModelOnline={isModelOnline} + showOnlyDisagreements={showOnlyDisagreements} + onToggleShowOnlyDisagreements={toggleShowOnlyDisagreements} /> {/* Main chart area */}
+ {/* Loading overlay for predictions */} + {predictionState.isLoading && ( +
+
+
+
+ Loading predictions... +
+
+
+ )}
diff --git a/src/components/CandleChart.tsx b/src/components/CandleChart.tsx index b94f360..8e07588 100644 --- a/src/components/CandleChart.tsx +++ b/src/components/CandleChart.tsx @@ -5,7 +5,7 @@ import { createChart, IChartApi, ISeriesApi, CandlestickData, HistogramData, Tim import { useTheme } from 'next-themes'; import SvgOverlay from './SvgOverlay'; import SpanAnnotationManager from './SpanAnnotationManager'; -import type { PerCandlePrediction, PredictionSpan, ModelInfoResponse } from '@/types/predictions'; +import type { PerCandlePrediction, PredictionSpan, ModelInfoResponse, PredictionSummary } from '@/types/predictions'; interface Candle { time: number; @@ -82,6 +82,8 @@ interface CandleChartProps { confidenceThreshold?: number; selectedLabels?: Set; modelInfo?: ModelInfoResponse | null; + predictionSummary?: PredictionSummary | null; + showOnlyDisagreements?: boolean; } export interface CandleChartHandle { @@ -108,6 +110,8 @@ const CandleChart = forwardRef( confidenceThreshold = 0.5, selectedLabels = new Set(), modelInfo = null, + predictionSummary = null, + showOnlyDisagreements = false, }, ref) => { const chartContainerRef = useRef(null); const chartRef = useRef(null); diff --git a/src/components/PredictionPanel.tsx b/src/components/PredictionPanel.tsx index 2a3c7e0..5f032c1 100644 --- a/src/components/PredictionPanel.tsx +++ b/src/components/PredictionPanel.tsx @@ -1,6 +1,5 @@ 'use client'; -import { useState } from 'react'; import type { PredictionState, ModelInfoResponse, PredictionSummary } from '@/types/predictions'; interface PredictionPanelProps { @@ -12,6 +11,8 @@ interface PredictionPanelProps { onToggleLabelSelection: (label: string) => void; predictionSummary?: PredictionSummary; isModelOnline: boolean; + showOnlyDisagreements?: boolean; + onToggleShowOnlyDisagreements?: () => void; } export default function PredictionPanel({ @@ -23,8 +24,9 @@ export default function PredictionPanel({ onToggleLabelSelection, predictionSummary, isModelOnline, + showOnlyDisagreements = false, + onToggleShowOnlyDisagreements, }: PredictionPanelProps) { - const [showOnlyDisagreements, setShowOnlyDisagreements] = useState(false); const { visible, @@ -175,13 +177,13 @@ export default function PredictionPanel({ )} {/* Disagreement Filter */} - {predictionSummary && predictionSummary.disagreements.length > 0 && ( + {predictionSummary && predictionSummary.disagreements.length > 0 && onToggleShowOnlyDisagreements && (