From e77dbe1ae70b6af4a819405e4cd01760a6064231 Mon Sep 17 00:00:00 2001 From: fciaf420 Date: Tue, 7 Oct 2025 18:16:18 -0400 Subject: [PATCH 1/5] feat: add configuration optimizer with UI and critical bug fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## New Features - Full-featured optimizer UI in main dashboard - Real-time progress tracking with WebSocket integration - Apply optimized settings directly from UI - Reset button to clear cached optimization results - Interactive weight sliders for PnL/Sharpe/Drawdown optimization ## Components Added - OptimizerButton: Main trigger button with inline progress - OptimizerDialog: Full optimization interface with tabs - OptimizerProgressBar: Real-time progress with ETA - OptimizerResults: Before/after comparison view - OptimizerWeightSliders: Interactive weight configuration ## API Routes - /api/optimizer/run: Start optimization job - /api/optimizer/status: Poll job progress - /api/optimizer/apply: Apply optimized config - /api/optimizer/cancel: Cancel running job - /api/optimizer/reset: Clear cached results ## Critical Bug Fix - Fixed optimizer script not applying symbol-specific settings - optimizedConfig.symbols now properly updated with optimized values - Previously only maxOpenPositions was applied, now ALL settings are applied - Adds optimized thresholds, trade sizes, leverage, TP/SL, time windows, cooldowns ## Backend Services - optimizerService: Job management with disk persistence - Survives route reloads and server restarts - Automatic config reload in UI after applying changes ## Configuration - Added optimizer dependencies to package.json - Updated .gitignore for backup files and optimizer cache 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .gitignore | 2 + optimize-config.js | 2388 +++++++++++++++++ package-lock.json | 22 + package.json | 5 +- src/app/api/optimizer/apply/route.ts | 62 + src/app/api/optimizer/cancel/route.ts | 59 + src/app/api/optimizer/reset/route.ts | 38 + src/app/api/optimizer/run/route.ts | 107 + src/app/api/optimizer/status/route.ts | 78 + src/components/optimizer/OptimizerButton.tsx | 201 ++ src/components/optimizer/OptimizerDialog.tsx | 330 +++ .../optimizer/OptimizerInfoTooltip.tsx | 72 + .../optimizer/OptimizerProgressBar.tsx | 279 ++ src/components/optimizer/OptimizerResults.tsx | 254 ++ .../optimizer/OptimizerWeightSliders.tsx | 114 + src/lib/services/optimizerService.ts | 449 ++++ 16 files changed, 4459 insertions(+), 1 deletion(-) create mode 100644 optimize-config.js create mode 100644 src/app/api/optimizer/apply/route.ts create mode 100644 src/app/api/optimizer/cancel/route.ts create mode 100644 src/app/api/optimizer/reset/route.ts create mode 100644 src/app/api/optimizer/run/route.ts create mode 100644 src/app/api/optimizer/status/route.ts create mode 100644 src/components/optimizer/OptimizerButton.tsx create mode 100644 src/components/optimizer/OptimizerDialog.tsx create mode 100644 src/components/optimizer/OptimizerInfoTooltip.tsx create mode 100644 src/components/optimizer/OptimizerProgressBar.tsx create mode 100644 src/components/optimizer/OptimizerResults.tsx create mode 100644 src/components/optimizer/OptimizerWeightSliders.tsx create mode 100644 src/lib/services/optimizerService.ts diff --git a/.gitignore b/.gitignore index 141abd0..341d923 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,6 @@ data/*.db-wal # user configuration config.user.json +config.user.backup*.json config.json +data/optimizer-jobs.json diff --git a/optimize-config.js b/optimize-config.js new file mode 100644 index 0000000..2c00c1c --- /dev/null +++ b/optimize-config.js @@ -0,0 +1,2388 @@ +#!/usr/bin/env node + +const Database = require('better-sqlite3'); +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const crypto = require('crypto'); +const readline = require('readline'); + +// API request configuration +const API_TIMEOUT_MS = 10000; // 10 second timeout +const MAX_RETRIES = 3; + +const FORCE_OPTIMIZER_OVERWRITE = process.env.FORCE_OPTIMIZER_OVERWRITE === '1'; +const FORCE_OPTIMIZER_CONFIRM = process.env.FORCE_OPTIMIZER_CONFIRM === '1'; + +// Realistic Slippage Model - Based on actual bot behavior +const EXIT_SLIPPAGE = { + TP: 0.0010, // 0.10% - TAKE_PROFIT_MARKET fills slightly worse than trigger + SL: 0.0050, // 0.50% - STOP_MARKET normal conditions (conservative baseline) + SL_VOLATILE: 0.0080, // 0.80% - STOP_MARKET during high volatility/cascades + ENTRY_LIMIT: 0.0000, // 0% - LIMIT orders don't slip (wait for fill at exact price) + ENTRY_MARKET: 0.0020 // 0.20% - MARKET fallback orders (10% of entries) +}; + +const LIMIT_FILL_RATE = 0.85; // 85% of LIMIT orders actually fill (15% miss due to price movement) +const MARKET_FALLBACK_RATE = 0.10; // 10% of entries use MARKET orders instead of LIMIT + +// Commission Model - Based on actual trading costs +const COMMISSION = { + MAKER_FEE: 0.0002, // 0.02% maker fee (LIMIT orders) + TAKER_FEE: 0.0004, // 0.04% taker fee (MARKET orders) + AVG_FILLS_PER_TRADE: 1.5 // Average fills per complete trade (entry + exit, small partial fills) + // Chunking only happens on large orders (> maxQty) + // For typical trade sizes ($10-25), minimal chunking occurs +}; + +const DEFAULT_SCORING_WEIGHTS = { + pnl: 50, + sharpe: 30, + drawdown: 20 +}; + +function parseScoringWeights() { + const parseWeight = (value, fallback) => { + if (value === undefined || value === null || value === '') { + return fallback; + } + + const numeric = Number(value); + if (!Number.isFinite(numeric) || numeric < 0) { + return fallback; + } + + return numeric; + }; + + const percent = { + pnl: parseWeight(process.env.OPTIMIZER_WEIGHT_PNL, DEFAULT_SCORING_WEIGHTS.pnl), + sharpe: parseWeight(process.env.OPTIMIZER_WEIGHT_SHARPE, DEFAULT_SCORING_WEIGHTS.sharpe), + drawdown: parseWeight(process.env.OPTIMIZER_WEIGHT_DRAWDOWN, DEFAULT_SCORING_WEIGHTS.drawdown) + }; + + const total = percent.pnl + percent.sharpe + percent.drawdown; + + if (total <= 0) { + const fallbackTotal = DEFAULT_SCORING_WEIGHTS.pnl + DEFAULT_SCORING_WEIGHTS.sharpe + DEFAULT_SCORING_WEIGHTS.drawdown; + return { + percent: { ...DEFAULT_SCORING_WEIGHTS }, + normalized: { + pnl: DEFAULT_SCORING_WEIGHTS.pnl / fallbackTotal, + sharpe: DEFAULT_SCORING_WEIGHTS.sharpe / fallbackTotal, + drawdown: DEFAULT_SCORING_WEIGHTS.drawdown / fallbackTotal + }, + isDefault: true + }; + } + + return { + percent, + normalized: { + pnl: percent.pnl / total, + sharpe: percent.sharpe / total, + drawdown: percent.drawdown / total + }, + isDefault: false + }; +} + +const scoringWeights = parseScoringWeights(); +const normalizedScoringWeights = scoringWeights.normalized; + +const formatWeightPercent = (value) => { + if (!Number.isFinite(value)) { + return '0%'; + } + + const rounded = Number(value.toFixed(1)); + if (Number.isInteger(rounded)) { + return `${Math.trunc(rounded)}%`; + } + + return `${rounded.toFixed(1)}%`; +}; + +// Connect to the database +const dbPath = path.join(__dirname, 'data', 'liquidations.db'); +const db = new Database(dbPath, { readonly: true }); + +// Load current configuration +const configPath = path.join(__dirname, 'config.user.json'); +const config = JSON.parse(fs.readFileSync(configPath, 'utf8')); + +// API helper functions for balance fetching +function buildSignedQuery(params, credentials) { + const timestamp = Date.now(); + const queryString = new URLSearchParams({ + ...params, + timestamp, + recvWindow: 5000 + }).toString(); + + const signature = crypto + .createHmac('sha256', credentials.secretKey) + .update(queryString) + .digest('hex'); + + return `${queryString}&signature=${signature}`; +} + +async function getAccountBalance(credentials) { + try { + const queryString = buildSignedQuery({}, credentials); + const response = await axios.get( + `https://fapi.asterdex.com/fapi/v2/balance?${queryString}`, + { + headers: { 'X-MBX-APIKEY': credentials.apiKey } + } + ); + + const usdtBalance = response.data.find(asset => asset.asset === 'USDT'); + return { + totalWalletBalance: parseFloat(usdtBalance?.walletBalance || 0), + availableBalance: parseFloat(usdtBalance?.availableBalance || 0), + crossMargin: parseFloat(usdtBalance?.crossUnPnl || 0) + }; + } catch (error) { + console.error('??? Failed to fetch balance:', error.response?.data || error.message); + return { totalWalletBalance: 0, availableBalance: 0, crossMargin: 0 }; + } +} + +async function getAccountInfo(credentials) { + try { + const queryString = buildSignedQuery({}, credentials); + const response = await axios.get( + `https://fapi.asterdex.com/fapi/v2/account?${queryString}`, + { + headers: { 'X-MBX-APIKEY': credentials.apiKey } + } + ); + + return response.data; + } catch (error) { + console.error('??? Failed to fetch account info:', error.response?.data || error.message); + return null; + } +} + +async function getUserTrades(credentials, symbol, limit = 100, startTime = null, endTime = null) { + try { + const params = { symbol, limit }; + if (startTime) params.startTime = startTime; + if (endTime) params.endTime = endTime; + + const queryString = buildSignedQuery(params, credentials); + const response = await axios.get( + `https://fapi.asterdex.com/fapi/v1/userTrades?${queryString}`, + { + headers: { 'X-MBX-APIKEY': credentials.apiKey } + } + ); + + return response.data; + } catch (error) { + console.error(`??? Failed to fetch trade history for ${symbol}:`, error.response?.data || error.message); + return []; + } +} + +async function getCurrentPositions(credentials) { + try { + const queryString = buildSignedQuery({}, credentials); + const response = await axios.get( + `https://fapi.asterdex.com/fapi/v2/positionRisk?${queryString}`, + { + headers: { 'X-MBX-APIKEY': credentials.apiKey } + } + ); + + // Filter out positions with zero size + const activePositions = response.data.filter(pos => parseFloat(pos.positionAmt) !== 0); + return activePositions; + } catch (error) { + console.error('??? Failed to fetch positions:', error.response?.data || error.message); + return []; + } +} + +// Retry wrapper for API calls with timeout +async function retryWithTimeout(fn, retries = MAX_RETRIES, timeoutMs = API_TIMEOUT_MS) { + for (let attempt = 1; attempt <= retries; attempt++) { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + + const result = await fn(controller.signal); + clearTimeout(timeoutId); + return result; + } catch (error) { + const isLastAttempt = attempt === retries; + + if (isLastAttempt) { + throw error; + } + + // Wait before retry (exponential backoff: 1s, 2s, 4s) + const delayMs = Math.min(1000 * Math.pow(2, attempt - 1), 5000); + await new Promise(resolve => setTimeout(resolve, delayMs)); + } + } +} + +// NEW: Fetch historical price data (klines) +async function _getHistoricalPrices(symbol, interval = '1m', limit = 1000) { + try { + const response = await axios.get( + `https://fapi.asterdex.com/fapi/v1/klines?symbol=${symbol}&interval=${interval}&limit=${limit}` + ); + + // Convert kline data to price points + const priceData = response.data.map(kline => ({ + timestamp: kline[0], // Open time + open: parseFloat(kline[1]), // Open price + high: parseFloat(kline[2]), // High price + low: parseFloat(kline[3]), // Low price + close: parseFloat(kline[4]), // Close price + volume: parseFloat(kline[5]) // Volume + })); + + return priceData; + } catch (error) { + console.error(`??? Failed to fetch price data for ${symbol}:`, error.response?.data || error.message); + return []; + } +} + +const MAX_KLINE_LIMIT = 1500; +const priceDataCache = new Map(); + +async function getCachedHistoricalPrices(symbol, interval = '1m', totalCandles = interval === '1m' ? 10080 : 1000) { + const cacheKey = `${symbol}:${interval}:${totalCandles}`; + if (priceDataCache.has(cacheKey)) { + return priceDataCache.get(cacheKey); + } + + const collected = []; + let remaining = Math.max(totalCandles, 0); + let endTime = undefined; + + while (remaining > 0) { + const requestLimit = Math.min(remaining, MAX_KLINE_LIMIT); + const params = new URLSearchParams({ + symbol, + interval, + limit: requestLimit + }); + + if (endTime) { + params.append('endTime', endTime); + } + + let response; + try { + response = await retryWithTimeout(async (signal) => { + return await axios.get(`https://fapi.asterdex.com/fapi/v1/klines?${params.toString()}`, { + timeout: API_TIMEOUT_MS, + signal + }); + }); + } catch (error) { + console.error(`??? Failed to fetch price data for ${symbol} after ${MAX_RETRIES} retries:`, error.response?.data || error.message); + break; + } + + const rawKlines = Array.isArray(response.data) ? response.data : []; + if (rawKlines.length === 0) { + break; + } + + const chunk = rawKlines.map(kline => ({ + timestamp: kline[0], + open: parseFloat(kline[1]), + high: parseFloat(kline[2]), + low: parseFloat(kline[3]), + close: parseFloat(kline[4]), + volume: parseFloat(kline[5]) + })); + + // Prepend older chunks so the array stays chronological from oldest to newest + collected.unshift(...chunk); + + const earliestTimestamp = chunk[0]?.timestamp; + if (typeof earliestTimestamp === 'number') { + endTime = earliestTimestamp - 1; + } else { + break; + } + + if (rawKlines.length < requestLimit) { + break; // No more data available + } + + remaining = Math.max(totalCandles - collected.length, 0); + } + + // Keep only the most recent `totalCandles` entries, preserving chronological order + const priceData = collected.length > totalCandles + ? collected.slice(collected.length - totalCandles) + : collected; + + priceDataCache.set(cacheKey, priceData); + return priceData; +} + +console.log('???? LIQUIDATION BOT OPTIMIZATION TOOL'); +console.log('====================================\n'); + +// Check liquidation price data coverage +function analyzePriceDataCoverage() { + console.log('???? PRICE DATA COVERAGE ANALYSIS'); + console.log('===============================\n'); + + const priceData = db.prepare(` + SELECT + event_time, + price, + volume_usdt, + side + FROM liquidations + WHERE symbol = 'ASTERUSDT' + ORDER BY event_time + LIMIT 10 + `).all(); + + console.log('Sample ASTERUSDT liquidation prices:'); + console.log('Time (ms) | Price | Volume | Side | Gap (min)'); + console.log('-----------------|---------|---------|------|----------'); + + let lastTime = 0; + priceData.forEach((row, i) => { + const gap = i > 0 ? (row.event_time - lastTime) / 1000 / 60 : 0; + console.log(`${row.event_time.toString().padEnd(16)} | $${row.price.toFixed(4)} | $${row.volume_usdt.toFixed(0).padEnd(7)} | ${row.side.padEnd(4)} | ${gap.toFixed(1)}min`); + lastTime = row.event_time; + }); + + // Check total coverage + const coverage = db.prepare(` + SELECT + COUNT(*) as total_events, + MIN(event_time) as first_time, + MAX(event_time) as last_time, + AVG(price) as avg_price, + MIN(price) as min_price, + MAX(price) as max_price + FROM liquidations + WHERE symbol = 'ASTERUSDT' + `).get(); + + const timeSpan = (coverage.last_time - coverage.first_time) / 1000 / 60 / 60; // hours + const avgGap = timeSpan * 60 / coverage.total_events; // minutes per event + + console.log(); + console.log(`???? ASTERUSDT Price Coverage:`); + console.log(` Total Events: ${coverage.total_events}`); + console.log(` Time Span: ${timeSpan.toFixed(1)} hours`); + console.log(` Average Gap: ${avgGap.toFixed(1)} minutes between price points`); + console.log(` Price Range: $${coverage.min_price.toFixed(4)} - $${coverage.max_price.toFixed(4)}`); + console.log(); + + return avgGap; +} + +// Helper functions +function formatNumber(num) { + return new Intl.NumberFormat('en-US', { + minimumFractionDigits: 2, + maximumFractionDigits: 2 + }).format(num); +} + +function formatLargeNumber(num) { + if (num >= 1000000) { + return (num / 1000000).toFixed(2) + 'M'; + } else if (num >= 1000) { + return (num / 1000).toFixed(1) + 'K'; + } + return num.toFixed(2); +} + +function formatCurrency(num) { + return Number.isFinite(num) ? `$${formatNumber(num)}` : 'n/a'; +} + +const DAY_MS = 24 * 60 * 60 * 1000; +const DEFAULT_THRESHOLD_WINDOW_MS = 60 * 1000; +const DEFAULT_THRESHOLD_COOLDOWN_MS = 30 * 1000; +const HUNTER_COOLDOWN_MS = 2 * 60 * 1000; + +function generateTimeWindowCandidates(currentMs) { + const seconds = Math.max(10, Math.round((currentMs || DEFAULT_THRESHOLD_WINDOW_MS) / 1000)); + const base = [20, 30, 45, 60, 75, 90, 120, 150, 180, 240]; + const dynamic = [ + seconds, + Math.max(10, Math.round(seconds * 0.5)), + Math.max(10, Math.round(seconds * 0.75)), + Math.round(seconds * 1.25), + Math.round(seconds * 1.5), + Math.round(seconds * 2) + ]; + + const candidates = new Set([...base, ...dynamic]); + const filtered = [...candidates] + .filter((sec) => Number.isFinite(sec) && sec >= 10 && sec <= 300) + .sort((a, b) => a - b); + + return filtered.map((sec) => sec * 1000); +} + +function generateCooldownCandidates(currentMs) { + const seconds = Math.max(5, Math.round((currentMs || DEFAULT_THRESHOLD_COOLDOWN_MS) / 1000)); + const base = [5, 10, 15, 20, 30, 45, 60, 90, 120, 180]; + const dynamic = [ + seconds, + Math.max(5, Math.round(seconds * 0.5)), + Math.max(5, Math.round(seconds * 0.75)), + Math.round(seconds * 1.25), + Math.round(seconds * 1.5) + ]; + + const candidates = new Set([...base, ...dynamic]); + const filtered = [...candidates] + .filter((sec) => Number.isFinite(sec) && sec >= 5 && sec <= 240) + .sort((a, b) => a - b); + + return filtered.map((sec) => sec * 1000); +} + +function calculateCombinationScore(longResult, shortResult) { + const combinedPnl = (longResult?.totalPnl || 0) + (shortResult?.totalPnl || 0); + const rawLongSharpe = longResult?.sharpeRatio ?? 0; + const rawShortSharpe = shortResult?.sharpeRatio ?? 0; + const cappedLongSharpe = Number.isFinite(rawLongSharpe) ? Math.min(Math.max(rawLongSharpe, -5), 5) : 0; + const cappedShortSharpe = Number.isFinite(rawShortSharpe) ? Math.min(Math.max(rawShortSharpe, -5), 5) : 0; + const combinedSharpe = (cappedLongSharpe + cappedShortSharpe) / 2; + const combinedDrawdown = Math.max(longResult?.maxDrawdown || 1, shortResult?.maxDrawdown || 1); + const drawdownScore = combinedPnl / (combinedDrawdown + 1); + + const finalScore = ( + (combinedPnl * normalizedScoringWeights.pnl) + + (combinedSharpe * normalizedScoringWeights.sharpe) + + (drawdownScore * normalizedScoringWeights.drawdown) + ); + + return { + finalScore, + combinedPnl, + combinedSharpe, + drawdownScore, + }; +} +const symbolSpanCache = new Map(); + + +function dedupeAndSort(values) { + return Array.from(new Set(values.filter(v => Number.isFinite(v) && v > 0))).sort((a, b) => a - b); +} + +function sampleCandidates(values, maxCount) { + const sorted = dedupeAndSort(values); + if (sorted.length <= maxCount) { + return sorted; + } + + const result = []; + const step = (sorted.length - 1) / (maxCount - 1); + for (let i = 0; i < maxCount; i++) { + const index = Math.round(i * step); + result.push(sorted[index]); + } + + return dedupeAndSort(result); +} + +function computePercentiles(values, percentiles) { + if (!values.length) return {}; + + const sorted = [...values].sort((a, b) => a - b); + const results = {}; + percentiles.forEach(p => { + if (p <= 0) { + results[p] = sorted[0]; + return; + } + if (p >= 1) { + results[p] = sorted[sorted.length - 1]; + return; + } + const index = (sorted.length - 1) * p; + const lower = Math.floor(index); + const upper = Math.ceil(index); + if (lower === upper) { + results[p] = sorted[lower]; + } else { + const weight = index - lower; + results[p] = sorted[lower] * (1 - weight) + sorted[upper] * weight; + } + }); + return results; +} + +function getLiquidationVolumes(symbol, side) { + const rows = db.prepare(` + SELECT volume_usdt + FROM liquidations + WHERE symbol = ? AND side = ? + `).all(symbol, side); + + return rows.map(row => parseFloat(row.volume_usdt) || 0).filter(v => v > 0); +} + +function generateThresholdCandidates(symbol, side, currentThreshold) { + const volumes = getLiquidationVolumes(symbol, side); + if (volumes.length === 0) { + return currentThreshold ? [currentThreshold] : []; + } + + const percentiles = computePercentiles(volumes, [0.5, 0.65, 0.75, 0.85, 0.9, 0.95, 0.98]); + const candidates = [currentThreshold]; + Object.values(percentiles).forEach(value => { + if (value && value > 0) { + // Round to nearest 10 for stability + candidates.push(Math.round(value / 10) * 10); + } + }); + + if (currentThreshold) { + candidates.push(currentThreshold * 0.75); + candidates.push(currentThreshold * 0.5); + candidates.push(currentThreshold * 1.25); + candidates.push(currentThreshold * 1.5); + } + + return dedupeAndSort(candidates); +} + +function computePriceVolatility(priceData) { + if (!priceData || priceData.length < 2) { + return { + avgAbsReturn: 0.5, + perc90: 1, + perc95: 1.5 + }; + } + + const returns = []; + for (let i = 1; i < priceData.length; i++) { + const prev = priceData[i - 1].close; + const curr = priceData[i].close; + if (prev > 0) { + const changePct = Math.abs(((curr - prev) / prev) * 100); + if (Number.isFinite(changePct)) { + returns.push(changePct); + } + } + } + + if (!returns.length) { + return { + avgAbsReturn: 0.5, + perc90: 1, + perc95: 1.5 + }; + } + + const avgAbsReturn = returns.reduce((sum, val) => sum + val, 0) / returns.length; + const percentileValues = computePercentiles(returns, [0.9, 0.95]); + + return { + avgAbsReturn, + perc90: percentileValues[0.9] || avgAbsReturn, + perc95: percentileValues[0.95] || percentileValues[0.9] || avgAbsReturn + }; +} + +function generateTpCandidates(volStats, currentTp) { + const base = Math.max(volStats.avgAbsReturn || 0.3, 0.1); + const highVol = Math.max(volStats.perc95 || base * 2, base); + const midVol = Math.max(volStats.perc90 || base, base); + + const anchors = Number.isFinite(currentTp) && currentTp > 0 + ? [currentTp, currentTp * 0.5, currentTp * 0.75, currentTp * 1.25, currentTp * 1.5, currentTp * 2] + : []; + + const general = [0.1, 0.15, 0.2, 0.25, 0.35, 0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5, 7.5, 10]; // trimmed: 12.5, 15, 20, 25, 30 + const dynamic = [ + base * 0.5, + base * 0.75, + base, + base * 1.25, + base * 1.5, + midVol, + highVol, + highVol * 1.5, + highVol * 2 + ]; + + const rawCandidates = [...general, ...dynamic, ...anchors] + .map(val => Number.isFinite(val) ? parseFloat(val.toFixed(2)) : null) + .filter(val => typeof val === 'number' && val > 0.05 && val <= 40); + + const candidates = sampleCandidates(rawCandidates, 15) + .filter(val => val >= 0.1 && val <= 30); // Basic sanity bounds only + + return candidates; +} + +function generateSlCandidates(volStats, currentSl) { + const base = Math.max(volStats.perc95 || volStats.avgAbsReturn * 2 || currentSl || 1, 0.5); + + const anchors = Number.isFinite(currentSl) && currentSl > 0 + ? [currentSl, currentSl * 0.5, currentSl * 0.75, currentSl * 1.25, currentSl * 1.5, currentSl * 2, currentSl * 3] + : []; + + const general = [0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5, 6, 7.5]; // trimmed: 10, 12.5, 15, 20, 25, 30, 35, 40 + const dynamic = [ + base * 0.5, + base * 0.75, + base, + base * 1.25, + base * 1.5, + base * 2, + base * 3 + ]; + + const rawCandidates = [...general, ...dynamic, ...anchors] + .map(val => Number.isFinite(val) ? parseFloat(val.toFixed(2)) : null) + .filter(val => typeof val === 'number' && val > 0.1 && val <= 80); + + const candidates = sampleCandidates(rawCandidates, 15) + .filter(val => val >= 0.1 && val <= 40); // Basic sanity bounds only + + return candidates; +} + +function generateLeverageCandidates(currentLeverage) { + const baseCandidates = [currentLeverage, 5, 7.5, 10, 12.5, 15, 20, 25]; + return dedupeAndSort(baseCandidates).filter(val => val > 0 && val <= 25); +} + +function generateMarginCandidates(capitalBudget, currentMargin) { + const base = currentMargin > 0 ? currentMargin : capitalBudget * 0.5; + const candidates = [currentMargin, base * 0.75, base, base * 1.25, capitalBudget * 0.5, capitalBudget * 0.75, capitalBudget]; + const sanitized = dedupeAndSort(candidates).map(val => Math.min(val, capitalBudget)); + return sanitized.filter(val => val > 0); +} + +async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spanDays) { + const cloneConfig = { ...symbolConfig }; + + const baseLongTradeSize = symbolConfig.longTradeSize ?? symbolConfig.tradeSize ?? 20; + const baseShortTradeSize = symbolConfig.shortTradeSize ?? symbolConfig.tradeSize ?? baseLongTradeSize; + const baseTradeSize = baseLongTradeSize; + const currentMargin = symbolConfig.maxPositionMarginUSDT || Math.max(baseTradeSize * 5, 50); + const leverageCurrent = symbolConfig.leverage || 10; + const currentLongThreshold = symbolConfig.longVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const currentShortThreshold = symbolConfig.shortVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const currentTp = symbolConfig.tpPercent || 1; + const currentSl = symbolConfig.slPercent || 5; + const thresholdEnabled = symbolConfig.useThreshold !== false; + const currentTimeWindowMs = symbolConfig.thresholdTimeWindow || DEFAULT_THRESHOLD_WINDOW_MS; + const currentCooldownMs = symbolConfig.thresholdCooldown || DEFAULT_THRESHOLD_COOLDOWN_MS; + const timeWindowCandidates = thresholdEnabled + ? generateTimeWindowCandidates(currentTimeWindowMs) + : [currentTimeWindowMs]; + const cooldownCandidates = thresholdEnabled + ? generateCooldownCandidates(currentCooldownMs) + : [currentCooldownMs]; + + const longBasePositions = Math.max(1, Math.floor(currentMargin / (baseTradeSize || 1)) || 1); + const shortBasePositions = Math.max(1, Math.floor(currentMargin / (baseShortTradeSize || 1)) || 1); + + // Preload price data for volatility estimation + const priceData = await getCachedHistoricalPrices(symbol, '1m', 10080); + const volStats = computePriceVolatility(priceData); + + const longThresholdCandidates = generateThresholdCandidates(symbol, 'SELL', currentLongThreshold || 1000); + const shortThresholdCandidates = generateThresholdCandidates(symbol, 'BUY', currentShortThreshold || 1000); + + const tpCandidatesFull = generateTpCandidates(volStats, currentTp); + const slCandidatesFull = generateSlCandidates(volStats, currentSl); + const tpCandidates = tpCandidatesFull.length > 10 + ? [...tpCandidatesFull.slice(0, 5), ...tpCandidatesFull.slice(-5)] + : tpCandidatesFull; + const slCandidates = slCandidatesFull.length > 10 + ? [...slCandidatesFull.slice(0, 5), ...slCandidatesFull.slice(-5)] + : slCandidatesFull; + const leverageCandidates = generateLeverageCandidates(leverageCurrent); // Test all leverage values + const marginCandidates = generateMarginCandidates(capitalBudget, currentMargin).slice(-6); // focus on higher budgets + + const backtestCache = new Map(); + const defaultCooldownMs = Math.max(0, currentCooldownMs); + const defaultWindowMs = Math.max(5_000, currentTimeWindowMs || DEFAULT_THRESHOLD_WINDOW_MS); + const defaultHunterCooldownMs = HUNTER_COOLDOWN_MS; + + const runBacktest = async (side, threshold, maxPositions, tradeSize, leverage, tp, sl, overrides = {}) => { + const { + cooldownMs = defaultCooldownMs, + hunterCooldownMs = defaultHunterCooldownMs, + windowMs = defaultWindowMs + } = overrides; + + const key = [side, threshold, maxPositions, tradeSize, leverage, tp, sl, cooldownMs, hunterCooldownMs, windowMs] + .map(v => Number.isFinite(v) ? Number(v).toFixed(6) : v) + .join('|'); + + if (backtestCache.has(key)) { + return backtestCache.get(key); + } + + const result = await backtestSymbol( + symbol, + side, + Math.max(1, Math.round(threshold)), + maxPositions, + tradeSize, + leverage, + tp, + sl, + { + suppressLogs: true, + cooldownMs, + hunterCooldownMs, + windowMs + } + ); + + backtestCache.set(key, result); + return result; + }; + + // Baseline performance + const currentLongBacktest = await runBacktest('SELL', Math.max(1, currentLongThreshold), longBasePositions, baseTradeSize, leverageCurrent, currentTp, currentSl); + const currentShortBacktest = await runBacktest('BUY', Math.max(1, currentShortThreshold), shortBasePositions, baseShortTradeSize, leverageCurrent, currentTp, currentSl); + const currentTotalPnl = currentLongBacktest.totalPnl + currentShortBacktest.totalPnl; + const dailyFactor = spanDays > 0 ? 1 / spanDays : 1; + const currentDailyPnl = currentTotalPnl * dailyFactor; + + // Calculate initial scores for current config + const rawCurrentLongSharpe = currentLongBacktest.sharpeRatio || 0; + const rawCurrentShortSharpe = currentShortBacktest.sharpeRatio || 0; + const cappedCurrentLongSharpe = Number.isFinite(rawCurrentLongSharpe) ? Math.min(Math.max(rawCurrentLongSharpe, -5), 5) : 0; + const cappedCurrentShortSharpe = Number.isFinite(rawCurrentShortSharpe) ? Math.min(Math.max(rawCurrentShortSharpe, -5), 5) : 0; + const currentSharpe = (cappedCurrentLongSharpe + cappedCurrentShortSharpe) / 2; + const currentDrawdown = Math.max(currentLongBacktest.maxDrawdown || 1, currentShortBacktest.maxDrawdown || 1); + const currentDrawdownScore = currentTotalPnl / (currentDrawdown + 1); +// Tri-factor scoring weights are configurable via optimizer UI (defaults: 50% PnL / 30% Sharpe / 20% Drawdown) + const currentFinalScore = (currentTotalPnl * 0.5) + (currentSharpe * 0.3) + (currentDrawdownScore * 0.2); + + let bestCombination = { + totalPnl: currentTotalPnl, + finalScore: currentFinalScore, + sharpeRatio: currentSharpe, + drawdownScore: currentDrawdownScore, + leverage: leverageCurrent, + margin: currentMargin, + tp: currentTp, + sl: currentSl, + long: { + threshold: Math.max(1, currentLongThreshold), + result: currentLongBacktest, + tradeSize: baseTradeSize, + maxPositions: longBasePositions + }, + short: { + threshold: Math.max(1, currentShortThreshold), + result: currentShortBacktest, + tradeSize: baseShortTradeSize, + maxPositions: shortBasePositions + } + }; + + for (const leverage of leverageCandidates) { + for (const margin of marginCandidates) { + if (!Number.isFinite(margin) || margin <= 0) continue; + + const longTradeSize = margin / longBasePositions; + const shortTradeSize = margin / shortBasePositions; + if (!Number.isFinite(longTradeSize) || longTradeSize <= 0) continue; + if (!Number.isFinite(shortTradeSize) || shortTradeSize <= 0) continue; + + for (const tp of tpCandidates) { + for (const sl of slCandidates) { + let bestLongSide = null; + for (const threshold of longThresholdCandidates) { + const candidateThreshold = Math.max(1, threshold); + const result = await runBacktest('SELL', candidateThreshold, longBasePositions, longTradeSize, leverage, tp, sl); + if (!bestLongSide || result.totalPnl > bestLongSide.result.totalPnl) { + bestLongSide = { + threshold: candidateThreshold, + result, + tradeSize: longTradeSize, + maxPositions: longBasePositions + }; + } + } + + let bestShortSide = null; + for (const threshold of shortThresholdCandidates) { + const candidateThreshold = Math.max(1, threshold); + const result = await runBacktest('BUY', candidateThreshold, shortBasePositions, shortTradeSize, leverage, tp, sl); + if (!bestShortSide || result.totalPnl > bestShortSide.result.totalPnl) { + bestShortSide = { + threshold: candidateThreshold, + result, + tradeSize: shortTradeSize, + maxPositions: shortBasePositions + }; + } + } + + if (!bestLongSide || !bestShortSide) continue; + + const combinedPnl = bestLongSide.result.totalPnl + bestShortSide.result.totalPnl; + const stopExitCount = (bestLongSide.result.exitReasons?.SL || 0) + (bestShortSide.result.exitReasons?.SL || 0); + const totalTrades = (bestLongSide.result.totalTrades || 0) + (bestShortSide.result.totalTrades || 0); + const stopRate = totalTrades > 0 ? stopExitCount / totalTrades : 0; + const combinedProfitFactor = ((bestLongSide.result.profitFactor || 0) + (bestShortSide.result.profitFactor || 0)) / 2; + + // Skip combinations with poor profit factor or excessive stop rate + if (combinedProfitFactor < 1.05 || stopRate > 0.65) { + continue; + } + + // CRITICAL: Liquidation distance check + // Liquidation occurs at approximately (100 / leverage)% price move + // Leave 10% safety margin for fees, funding, and slippage + const liquidationDistance = (100 / leverage) * 0.9; // 90% of theoretical distance + if (sl >= liquidationDistance) { + continue; // SL would never execute - position gets liquidated first! + } + + // Risk management constraint: Enforce minimum R:R ratio + // Reject if TP/SL < 0.33 (worse than 1:3 R:R - requires >75% win rate) + const riskRewardRatio = tp / sl; + if (riskRewardRatio < 0.33) { + continue; // Skip combinations with terrible R:R ratios + } + + // Calculate required win rate for profitability + // Required WR = SL / (TP + SL) + const requiredWinRate = sl / (tp + sl); + const combinedWinRate = ((bestLongSide.result.winRate || 0) + (bestShortSide.result.winRate || 0)) / 2 / 100; + + // Skip if backtest win rate is below required (with 5% safety margin) + if (combinedWinRate < requiredWinRate + 0.05) { + continue; // Not profitable enough even in optimistic backtest + } + + // Tri-factor weighted scoring system (weights configured via UI sliders) + // Factor 1: Total PnL - prioritizes profit and capital deployment + const pnlScore = combinedPnl; + + // Factor 2: Sharpe Ratio - monitors consistency + // Cap Sharpe at 5.0 to prevent infinity from unrealistic backtests + const rawLongSharpe = bestLongSide.result.sharpeRatio || 0; + const rawShortSharpe = bestShortSide.result.sharpeRatio || 0; + const cappedLongSharpe = Number.isFinite(rawLongSharpe) ? Math.min(Math.max(rawLongSharpe, -5), 5) : 0; + const cappedShortSharpe = Number.isFinite(rawShortSharpe) ? Math.min(Math.max(rawShortSharpe, -5), 5) : 0; + const combinedSharpe = (cappedLongSharpe + cappedShortSharpe) / 2; + + // Factor 3: PnL per Drawdown - keeps risk in check + const combinedDrawdown = Math.max(bestLongSide.result.maxDrawdown || 1, bestShortSide.result.maxDrawdown || 1); + const drawdownScore = combinedPnl / (combinedDrawdown + 1); // +1 to avoid division by zero + + // Calculate weighted final score using normalized weights from the optimizer configuration + const finalScore = ( + (pnlScore * normalizedScoringWeights.pnl) + + (combinedSharpe * normalizedScoringWeights.sharpe) + + (drawdownScore * normalizedScoringWeights.drawdown) + ); + + // Sanity check for NaN/Infinity + if (!Number.isFinite(finalScore)) { + continue; + } + + if (finalScore > bestCombination.finalScore) { + bestCombination = { + totalPnl: combinedPnl, + finalScore: finalScore, + sharpeRatio: combinedSharpe, + drawdownScore: drawdownScore, + leverage, + margin, + tp, + sl, + long: bestLongSide, + short: bestShortSide + }; + } + } + } + } + } + + let bestWindowMs = currentTimeWindowMs; + let bestCooldownMs = currentCooldownMs; + let cachedBestScore = bestCombination.finalScore; + let cachedBestLongResult = bestCombination.long.result; + let cachedBestShortResult = bestCombination.short.result; + + if (thresholdEnabled) { + for (const windowMs of timeWindowCandidates) { + for (const cooldownMs of cooldownCandidates) { + const longResult = await runBacktest( + 'SELL', + Math.max(1, bestCombination.long.threshold), + bestCombination.long.maxPositions, + bestCombination.long.tradeSize, + bestCombination.leverage, + bestCombination.tp, + bestCombination.sl, + { windowMs, cooldownMs } + ); + + const shortResult = await runBacktest( + 'BUY', + Math.max(1, bestCombination.short.threshold), + bestCombination.short.maxPositions, + bestCombination.short.tradeSize, + bestCombination.leverage, + bestCombination.tp, + bestCombination.sl, + { windowMs, cooldownMs } + ); + + const metrics = calculateCombinationScore(longResult, shortResult); + + if (metrics.finalScore > cachedBestScore + 1e-6) { + cachedBestScore = metrics.finalScore; + cachedBestLongResult = longResult; + cachedBestShortResult = shortResult; + bestWindowMs = windowMs; + bestCooldownMs = cooldownMs; + bestCombination.totalPnl = metrics.combinedPnl; + bestCombination.sharpeRatio = metrics.combinedSharpe; + bestCombination.drawdownScore = metrics.drawdownScore; + } + } + } + } + + bestCombination.finalScore = cachedBestScore; + bestCombination.long.result = cachedBestLongResult; + bestCombination.short.result = cachedBestShortResult; + bestCombination.windowMs = bestWindowMs; + bestCombination.cooldownMs = bestCooldownMs; + + const finalMetrics = calculateCombinationScore(bestCombination.long.result, bestCombination.short.result); + bestCombination.totalPnl = finalMetrics.combinedPnl; + bestCombination.sharpeRatio = finalMetrics.combinedSharpe; + bestCombination.drawdownScore = finalMetrics.drawdownScore; + bestCombination.finalScore = Math.max(bestCombination.finalScore, finalMetrics.finalScore); + + const optimizedDailyPnl = bestCombination.totalPnl * dailyFactor; + + const longImprovement = (bestCombination.long.result.totalPnl - currentLongBacktest.totalPnl) * dailyFactor; + const shortImprovement = (bestCombination.short.result.totalPnl - currentShortBacktest.totalPnl) * dailyFactor; + + const vwapOptimized = symbolConfig.vwapProtection === false + ? false + : (bestCombination.long.threshold < Math.max(1, currentLongThreshold) * 0.7 + || bestCombination.short.threshold < Math.max(1, currentShortThreshold) * 0.7 + ? false + : symbolConfig.vwapProtection); + + const optimizedSymbolConfig = { + ...cloneConfig, + longVolumeThresholdUSDT: Math.round(bestCombination.long.threshold), + shortVolumeThresholdUSDT: Math.round(bestCombination.short.threshold), + tradeSize: parseFloat((bestCombination.long.tradeSize).toFixed(2)), + longTradeSize: parseFloat((bestCombination.long.tradeSize).toFixed(2)), + shortTradeSize: parseFloat((bestCombination.short.tradeSize).toFixed(2)), + maxPositionMarginUSDT: parseFloat(bestCombination.margin.toFixed(2)), + leverage: bestCombination.leverage, + tpPercent: parseFloat(bestCombination.tp.toFixed(2)), + slPercent: parseFloat(bestCombination.sl.toFixed(2)), + vwapProtection: vwapOptimized, + thresholdTimeWindow: Math.round(bestCombination.windowMs || currentTimeWindowMs), + thresholdCooldown: Math.round(bestCombination.cooldownMs || currentCooldownMs) + }; + + return { + symbol, + current: { + longThreshold: Math.max(1, currentLongThreshold), + shortThreshold: Math.max(1, currentShortThreshold), + tradeSize: baseTradeSize, + longTradeSize: baseTradeSize, + shortTradeSize: baseShortTradeSize, + margin: currentMargin, + leverage: leverageCurrent, + tp: currentTp, + sl: currentSl, + longMaxPositions: longBasePositions, + shortMaxPositions: shortBasePositions, + thresholdTimeWindow: currentTimeWindowMs, + thresholdCooldown: currentCooldownMs, + performance: { + long: currentLongBacktest, + short: currentShortBacktest, + totalPnl: currentTotalPnl, + dailyPnl: currentDailyPnl + } + }, + optimized: { + config: optimizedSymbolConfig, + long: bestCombination.long, + short: bestCombination.short, + leverage: bestCombination.leverage, + thresholdTimeWindow: bestCombination.windowMs, + thresholdCooldown: bestCombination.cooldownMs, + tp: bestCombination.tp, + sl: bestCombination.sl, + totalPnl: bestCombination.totalPnl, + dailyPnl: optimizedDailyPnl, + finalScore: bestCombination.finalScore, + sharpeRatio: bestCombination.sharpeRatio, + drawdownScore: bestCombination.drawdownScore + }, + improvements: { + long: longImprovement, + short: shortImprovement, + totalDaily: optimizedDailyPnl - currentDailyPnl + }, + spanDays, + capitalBudget + }; +} + +function getSymbolDataSpanDays(symbol) { + if (symbolSpanCache.has(symbol)) { + return symbolSpanCache.get(symbol); + } + + const spanRow = db.prepare(` + SELECT MIN(event_time) as first_time, MAX(event_time) as last_time + FROM liquidations + WHERE symbol = ? + `).get(symbol); + + let spanDays = 0; + if (spanRow && typeof spanRow.first_time === 'number' && typeof spanRow.last_time === 'number' && spanRow.last_time > spanRow.first_time) { + spanDays = (spanRow.last_time - spanRow.first_time) / DAY_MS; + } + + const minimumSpan = 1 / 24; // Assume at least 1 hour of data to avoid division by zero + spanDays = Math.max(spanDays, minimumSpan); + symbolSpanCache.set(symbol, spanDays); + return spanDays; +} + +// Helper function to analyze 60-second rolling windows +function analyzeRollingWindows(symbol, side, threshold, windowSize = 60000) { + // Get all liquidations for this symbol and side, ordered by time + const liquidations = db.prepare(` + SELECT event_time, volume_usdt + FROM liquidations + WHERE symbol = ? AND side = ? + ORDER BY event_time + `).all(symbol, side); + + if (liquidations.length === 0) { + return { totalTriggers: 0, avgWindowVolume: 0, maxWindowVolume: 0, dailyTriggers: 0 }; + } + + let triggers = 0; + let windowVolumes = []; + + // For each liquidation, calculate the rolling 60-second window volume + for (let i = 0; i < liquidations.length; i++) { + const currentTime = liquidations[i].event_time; + const windowStart = currentTime - windowSize; + + // Find all liquidations within the 60-second window BEFORE and INCLUDING current + let windowVolume = 0; + + for (let j = i; j >= 0; j--) { + if (liquidations[j].event_time >= windowStart && liquidations[j].event_time <= currentTime) { + windowVolume += liquidations[j].volume_usdt; + } else if (liquidations[j].event_time < windowStart) { + break; // No need to go further back + } + } + + windowVolumes.push(windowVolume); + + // Check if this window crosses the threshold + if (windowVolume >= threshold) { + triggers++; + } + } + + const avgWindowVolume = windowVolumes.length > 0 + ? windowVolumes.reduce((a, b) => a + b, 0) / windowVolumes.length + : 0; + const maxWindowVolume = windowVolumes.length > 0 + ? Math.max(...windowVolumes) + : 0; + + const spanMs = liquidations[liquidations.length - 1].event_time - liquidations[0].event_time; + const spanDays = Math.max(spanMs / DAY_MS, 1 / 24); + + return { + totalTriggers: triggers, + avgWindowVolume, + maxWindowVolume, + dailyTriggers: spanDays > 0 ? triggers / spanDays : 0, + spanDays + }; +} + +// 1. Current Config Performance Analysis using Rolling Windows +function analyzeCurrentConfig() { + console.log('???? ROLLING 60-SECOND WINDOW ANALYSIS'); + console.log('=====================================\n'); + + for (const [symbol, symbolConfig] of Object.entries(config.symbols)) { + console.log(`???? ${symbol} Analysis:`); + + const timeWindowMs = symbolConfig.thresholdTimeWindow || DEFAULT_THRESHOLD_WINDOW_MS; + const longThreshold = symbolConfig.longVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const shortThreshold = symbolConfig.shortVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const tradeSize = symbolConfig.tradeSize || 20; + const leverage = symbolConfig.leverage || 10; + const tpPercent = symbolConfig.tpPercent || 1; + const profitPerTrade = tradeSize * leverage * (tpPercent / 100); + + // Analyze rolling windows for LONG opportunities (SELL liquidations) + const longAnalysis = analyzeRollingWindows(symbol, 'SELL', longThreshold); + const shortAnalysis = analyzeRollingWindows(symbol, 'BUY', shortThreshold); + + console.log(` ???? LONG Opportunities (${(timeWindowMs/1000).toFixed(0)}s rolling SELL liquidations):`); + console.log(` Threshold: $${formatLargeNumber(longThreshold)}`); + console.log(` Daily Triggers: ${longAnalysis.dailyTriggers.toFixed(1)}`); + console.log(` Daily Profit: $${formatLargeNumber(longAnalysis.dailyTriggers * profitPerTrade)}`); + console.log(` Avg Window Volume: $${formatLargeNumber(longAnalysis.avgWindowVolume)}`); + console.log(` Max Window Volume: $${formatLargeNumber(longAnalysis.maxWindowVolume)}`); + + console.log(` ???? SHORT Opportunities (${(timeWindowMs/1000).toFixed(0)}s rolling BUY liquidations):`); + console.log(` Threshold: $${formatLargeNumber(shortThreshold)}`); + console.log(` Daily Triggers: ${shortAnalysis.dailyTriggers.toFixed(1)}`); + console.log(` Daily Profit: $${formatLargeNumber(shortAnalysis.dailyTriggers * profitPerTrade)}`); + console.log(` Avg Window Volume: $${formatLargeNumber(shortAnalysis.avgWindowVolume)}`); + console.log(` Max Window Volume: $${formatLargeNumber(shortAnalysis.maxWindowVolume)}`); + + const totalDailyProfit = (longAnalysis.dailyTriggers + shortAnalysis.dailyTriggers) * profitPerTrade; + console.log(` ???? Total Daily Profit: $${formatLargeNumber(totalDailyProfit)}`); + console.log(); + } +} + +// 2. Threshold Optimization Analysis using Rolling Windows +function optimizeThresholds() { + console.log('???? ROLLING WINDOW THRESHOLD OPTIMIZATION'); + console.log('========================================\n'); + + // Focus on most active symbols + const topSymbols = ['ASTERUSDT', 'BTCUSDT', 'ETHUSDT', 'SOLUSDT']; + + topSymbols.forEach(symbol => { + if (!config.symbols[symbol]) return; + + console.log(`???? ${symbol} Rolling Window Optimization:`); + + const symbolConfig = config.symbols[symbol]; + const timeWindowMs = symbolConfig.thresholdTimeWindow || DEFAULT_THRESHOLD_WINDOW_MS; + const tradeSize = symbolConfig.tradeSize || 20; + const leverage = symbolConfig.leverage || 10; + const tpPercent = symbolConfig.tpPercent || 1; + const profitPerTrade = tradeSize * leverage * (tpPercent / 100); + const currentLong = symbolConfig.longVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const currentShort = symbolConfig.shortVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + + // Test different threshold levels + const thresholds = [1000, 2000, 3000, 5000, 8000, 10000, 15000, 20000, 25000, 50000]; + + console.log(` ???? LONG Opportunities (${(timeWindowMs/1000).toFixed(0)}s rolling SELL liquidations):`); + console.log(' Threshold | Daily Triggers | Daily Profit | Current'); + console.log(' ----------|----------------|--------------|--------'); + + thresholds.forEach(threshold => { + const analysis = analyzeRollingWindows(symbol, 'SELL', threshold, timeWindowMs); + const dailyProfit = analysis.dailyTriggers * profitPerTrade; + const isCurrent = threshold === currentLong ? ' <- CURRENT' : ''; + + console.log(` $${threshold.toString().padEnd(8)} | ${analysis.dailyTriggers.toFixed(1).padEnd(14)} | $${formatLargeNumber(dailyProfit).padEnd(12)} |${isCurrent}`); + }); + + console.log('\n ???? SHORT Opportunities (${(timeWindowMs/1000).toFixed(0)}s rolling BUY liquidations):'); + console.log(' Threshold | Daily Triggers | Daily Profit | Current'); + console.log(' ----------|----------------|--------------|--------'); + + thresholds.forEach(threshold => { + const analysis = analyzeRollingWindows(symbol, 'BUY', threshold, timeWindowMs); + const dailyProfit = analysis.dailyTriggers * profitPerTrade; + const isCurrent = threshold === currentShort ? ' <- CURRENT' : ''; + + console.log(` $${threshold.toString().padEnd(8)} | ${analysis.dailyTriggers.toFixed(1).padEnd(14)} | $${formatLargeNumber(dailyProfit).padEnd(12)} |${isCurrent}`); + }); + + console.log(); + }); +} + +// 3. Symbol Profitability Ranking +function rankSymbolProfitability() { + console.log('???? SYMBOL PROFITABILITY RANKING'); + console.log('-------------------------------\n'); + + const symbolStats = []; + + for (const [symbol, symbolConfig] of Object.entries(config.symbols)) { + const tradeSize = symbolConfig.tradeSize || 20; + const leverage = symbolConfig.leverage || 10; + const tpPercent = symbolConfig.tpPercent || 1; + const longThreshold = symbolConfig.longVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const shortThreshold = symbolConfig.shortVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + + const stats = db.prepare(` + SELECT + COUNT(CASE WHEN side = 'SELL' AND volume_usdt >= ? THEN 1 END) as long_triggers, + COUNT(CASE WHEN side = 'BUY' AND volume_usdt >= ? THEN 1 END) as short_triggers, + COUNT(CASE WHEN side = 'SELL' THEN 1 END) as total_long_opportunities, + COUNT(CASE WHEN side = 'BUY' THEN 1 END) as total_short_opportunities, + AVG(CASE WHEN side = 'SELL' THEN volume_usdt END) as avg_long_volume, + AVG(CASE WHEN side = 'BUY' THEN volume_usdt END) as avg_short_volume + FROM liquidations + WHERE symbol = ? + `).get(longThreshold, shortThreshold, symbol); + + if (!stats || (stats.long_triggers + stats.short_triggers) === 0) continue; + + const profitPerTrade = tradeSize * leverage * (tpPercent / 100); + const totalTriggers = stats.long_triggers + stats.short_triggers; + const estimatedProfit = totalTriggers * profitPerTrade; + const spanDays = getSymbolDataSpanDays(symbol); + const dailyTriggers = spanDays > 0 ? totalTriggers / spanDays : 0; + const dailyProfit = spanDays > 0 ? estimatedProfit / spanDays : 0; + + symbolStats.push({ + symbol, + totalTriggers, + estimatedProfit, + dailyTriggers, + dailyProfit, + profitPerTrade, + longCaptureRate: stats.total_long_opportunities > 0 + ? (stats.long_triggers / stats.total_long_opportunities * 100) + : 0, + shortCaptureRate: stats.total_short_opportunities > 0 + ? (stats.short_triggers / stats.total_short_opportunities * 100) + : 0 + }); + } + + // Sort by daily profit + symbolStats.sort((a, b) => b.dailyProfit - a.dailyProfit); + + console.log('Ranking by Daily Profit Potential:'); + console.log('Rank | Symbol | Daily Triggers | Daily Profit | Capture Rate | Profit/Trade'); + console.log('-----|-----------|----------------|--------------|--------------|-------------'); + + symbolStats.forEach((stat, i) => { + console.log(`${(i + 1).toString().padEnd(4)} | ${stat.symbol.padEnd(9)} | ${stat.dailyTriggers.toFixed(1).padEnd(14)} | $${formatLargeNumber(stat.dailyProfit).padEnd(11)} | ${((stat.longCaptureRate + stat.shortCaptureRate) / 2).toFixed(1)}%${' '.padEnd(8)} | $${formatNumber(stat.profitPerTrade)}`); + }); + + console.log(); +} + +// Helper function to calculate risk metrics +function calculateRiskMetrics(trades) { + if (trades.length === 0) { + return { sharpeRatio: 0, maxDrawdown: 0, maxDrawdownPercent: 0, profitFactor: 0 }; + } + + // Calculate returns for each trade + const returns = trades.map(t => t.pnl); + const avgReturn = returns.reduce((sum, r) => sum + r, 0) / returns.length; + + // Calculate standard deviation + const variance = returns.reduce((sum, r) => sum + Math.pow(r - avgReturn, 2), 0) / returns.length; + const stdDev = Math.sqrt(variance); + + // Sharpe Ratio (assuming risk-free rate of 0) + const sharpeRatio = stdDev > 0 ? avgReturn / stdDev : 0; + + // Maximum Drawdown + let peak = 0; + let maxDrawdown = 0; + let runningPnl = 0; + + for (const trade of trades) { + runningPnl += trade.pnl; + if (runningPnl > peak) { + peak = runningPnl; + } + const drawdown = peak - runningPnl; + if (drawdown > maxDrawdown) { + maxDrawdown = drawdown; + } + } + + const maxDrawdownPercent = peak > 0 ? (maxDrawdown / peak) * 100 : 0; + + // Profit Factor (total wins / total losses) + const totalWins = trades.filter(t => t.pnl > 0).reduce((sum, t) => sum + t.pnl, 0); + const totalLosses = Math.abs(trades.filter(t => t.pnl < 0).reduce((sum, t) => sum + t.pnl, 0)); + const profitFactor = totalLosses > 0 ? totalWins / totalLosses : (totalWins > 0 ? Infinity : 0); + + return { sharpeRatio, maxDrawdown, maxDrawdownPercent, profitFactor }; +} + +// REALISTIC BACKTEST ENGINE with Historical Price Integration +async function backtestSymbol(symbol, side, threshold, maxPositions, tradeSize, leverage, tpPercent, slPercent, options = {}) { + const suppressLogs = options.suppressLogs || false; + const cooldownMs = Number.isFinite(options.cooldownMs) ? Math.max(0, options.cooldownMs) : 0; + const hunterCooldownMs = Number.isFinite(options.hunterCooldownMs) ? Math.max(0, options.hunterCooldownMs) : HUNTER_COOLDOWN_MS; + const windowMs = Number.isFinite(options.windowMs) ? Math.max(1_000, options.windowMs) : DEFAULT_THRESHOLD_WINDOW_MS; + const log = (...args) => { + if (!suppressLogs) { + console.log(...args); + } + }; + + log(`???? Backtesting ${symbol} ${side === 'SELL' ? 'LONG' : 'SHORT'} with $${threshold} threshold...`); + + // Get liquidations ordered by time + const liquidations = db.prepare(` + SELECT event_time, volume_usdt, price + FROM liquidations + WHERE symbol = ? AND side = ? + ORDER BY event_time + `).all(symbol, side); + + if (liquidations.length === 0) { + return { + totalTrades: 0, wins: 0, losses: 0, totalPnl: 0, winRate: 0, + avgWin: 0, avgLoss: 0, avgDuration: 0, activePositions: 0, + sharpeRatio: 0, maxDrawdown: 0, maxDrawdownPercent: 0, profitFactor: 0 + }; + } + + // Get historical price data from API for accurate TP/SL simulation + let priceData = []; + try { + priceData = await getCachedHistoricalPrices(symbol, '1m', 10080); // Last ~7 days at 1-minute resolution + } catch (_error) { + log(` ?????? Could not fetch historical prices for ${symbol}, using liquidation prices`); + } + + // If we have historical price data, use it; otherwise fall back to liquidation prices + const allPrices = priceData.length > 0 + ? priceData.map(p => ({ event_time: p.timestamp, price: p.close, high: p.high, low: p.low })) + : db.prepare(`SELECT event_time, price, price as high, price as low FROM liquidations WHERE symbol = ? ORDER BY event_time`).all(symbol); + + let activePositions = []; + let completedTrades = []; + let totalPnl = 0; + let priceIndex = 0; + let lastEntryTime = -Infinity; + let lastHunterEntryTime = -Infinity; + + // Calculate volatility factor from recent price movements (for slippage adjustment) + const calculateVolatilityFactor = (currentIndex, lookbackPeriods = 20) => { + if (allPrices.length < 2) return 1.0; + + const startIndex = Math.max(0, currentIndex - lookbackPeriods); + const endIndex = Math.min(allPrices.length, currentIndex + 1); + const slice = allPrices.slice(startIndex, endIndex); + + if (slice.length < 2) return 1.0; + + // Calculate average absolute return + let sumAbsReturn = 0; + let count = 0; + for (let i = 1; i < slice.length; i++) { + const prev = slice[i - 1].price || slice[i - 1].close; + const curr = slice[i].price || slice[i].close; + if (prev > 0) { + sumAbsReturn += Math.abs((curr - prev) / prev); + count++; + } + } + + if (count === 0) return 1.0; + + const avgReturn = sumAbsReturn / count; + // Volatility factor: 1.0 = normal (0.5% avg move), scales up/down from there + return Math.max(0.5, Math.min(3.0, avgReturn / 0.005)); + }; + + const recordExit = (pos, exitPrice, exitReason, priceEventTime, volatilityFactor = 1.0) => { + // Apply realistic slippage based on order type and market conditions + let actualExitPrice = exitPrice; + + if (exitReason === 'TP') { + // TAKE_PROFIT_MARKET: fills slightly worse than trigger price + actualExitPrice = pos.isLong + ? exitPrice * (1 - EXIT_SLIPPAGE.TP) // LONG TP: sell fills lower + : exitPrice * (1 + EXIT_SLIPPAGE.TP); // SHORT TP: buy fills higher + } else if (exitReason === 'SL') { + // STOP_MARKET: worse slippage, especially in volatile conditions + const slippageRate = volatilityFactor > 1.5 ? EXIT_SLIPPAGE.SL_VOLATILE : EXIT_SLIPPAGE.SL; + actualExitPrice = pos.isLong + ? exitPrice * (1 - slippageRate) // LONG SL: sell fills even lower + : exitPrice * (1 + slippageRate); // SHORT SL: buy fills even higher + } + // EOD exits use actual exit price (no slippage) + + // Calculate gross PnL before commissions + const grossPnl = pos.isLong + ? (actualExitPrice - pos.entryPrice) * pos.size + : (pos.entryPrice - actualExitPrice) * pos.size; + + // Calculate commission costs + // Entry: mostly LIMIT (maker fee), some MARKET (taker fee) + // Exit: TP uses TAKE_PROFIT_MARKET (taker), SL uses STOP_MARKET (taker) + const notional = tradeSize * leverage; + const entryCommission = notional * (COMMISSION.MAKER_FEE * 0.9 + COMMISSION.TAKER_FEE * 0.1); // 90% LIMIT, 10% MARKET + const exitCommission = exitReason === 'EOD' + ? notional * COMMISSION.MAKER_FEE // EOD might use LIMIT + : notional * COMMISSION.TAKER_FEE; // TP/SL use MARKET orders + + // Apply fill multiplier for order chunking (PositionManager splits orders) + const totalCommission = (entryCommission + exitCommission) * COMMISSION.AVG_FILLS_PER_TRADE; + + // Net PnL after commissions + const netPnl = grossPnl - totalCommission; + + totalPnl += netPnl; + completedTrades.push({ + symbol, + side: pos.isLong ? 'LONG' : 'SHORT', + entryPrice: pos.entryPrice, + exitPrice: actualExitPrice, + triggerPrice: exitPrice, // Store original trigger price for comparison + slippage: Math.abs(actualExitPrice - exitPrice), + grossPnl, // PnL before commissions + commission: totalCommission, // Total commission cost + pnl: netPnl, // Net PnL after commissions + exitReason, + duration: priceEventTime - pos.entryTime, + margin: tradeSize, + volatilityFactor: exitReason === 'SL' ? volatilityFactor : null + }); + }; + + const evaluatePositionsOnBar = (priceBar, barIndex) => { + const volatilityFactor = calculateVolatilityFactor(barIndex); + + activePositions = activePositions.filter(pos => { + let shouldExit = false; + let exitReason = null; + let exitPrice = null; + + // Check if both TP and SL were touched in this candle + const tpTouched = pos.isLong ? priceBar.high >= pos.tpPrice : priceBar.low <= pos.tpPrice; + const slTouched = pos.isLong ? priceBar.low <= pos.slPrice : priceBar.high >= pos.slPrice; + + if (tpTouched && slTouched) { + // Both touched - determine which hit first based on distance from entry + const tpDistance = Math.abs(pos.tpPrice - pos.entryPrice); + const slDistance = Math.abs(pos.slPrice - pos.entryPrice); + + // Probabilistic model: 70% of time, closer target hits first + // 30% of time, further target hits (price whipsaws) + const closerHitsFirst = Math.random() < 0.70; + + if (closerHitsFirst) { + if (slDistance < tpDistance) { + // SL is closer - assume it hit first (more realistic) + exitReason = 'SL'; + exitPrice = pos.slPrice; + } else { + // TP is closer - assume it hit first + exitReason = 'TP'; + exitPrice = pos.tpPrice; + } + } else { + // Whipsaw scenario - further target hits + if (slDistance < tpDistance) { + exitReason = 'TP'; + exitPrice = pos.tpPrice; + } else { + exitReason = 'SL'; + exitPrice = pos.slPrice; + } + } + shouldExit = true; + } else if (tpTouched) { + // Only TP touched + shouldExit = true; + exitReason = 'TP'; + exitPrice = pos.tpPrice; + } else if (slTouched) { + // Only SL touched + shouldExit = true; + exitReason = 'SL'; + exitPrice = pos.slPrice; + } + + if (shouldExit) { + recordExit(pos, exitPrice, exitReason, priceBar.event_time, volatilityFactor); + return false; + } + return true; + }); + }; + + // Process each liquidation event + for (let i = 0; i < liquidations.length; i++) { + const currentEvent = liquidations[i]; + const currentTime = currentEvent.event_time; + const windowStart = currentTime - windowMs; + + // Calculate rolling window volume + let windowVolume = 0; + for (let j = i; j >= 0; j--) { + if (liquidations[j].event_time >= windowStart && liquidations[j].event_time <= currentTime) { + windowVolume += liquidations[j].volume_usdt; + } else if (liquidations[j].event_time < windowStart) { + break; + } + } + + // Check for position exits using ALL price data between last check and now + while (priceIndex < allPrices.length && allPrices[priceIndex].event_time <= currentTime) { + const priceBar = allPrices[priceIndex]; + evaluatePositionsOnBar(priceBar, priceIndex); + priceIndex++; + } + + // Check if we can open new position + const cooldownElapsed = currentTime - lastEntryTime >= cooldownMs; + const hunterCooldownElapsed = currentTime - lastHunterEntryTime >= hunterCooldownMs; + if (windowVolume >= threshold && activePositions.length < maxPositions && cooldownElapsed && hunterCooldownElapsed) { + // Simulate LIMIT order non-fill probability (15% of orders don't fill) + if (Math.random() > LIMIT_FILL_RATE) { + // LIMIT order placed but never filled - skip this entry + if (!suppressLogs && Math.random() < 0.1) { // Log 10% of non-fills to avoid spam + log(` ???? LIMIT order did not fill (non-fill simulation)`); + } + continue; // Skip to next liquidation event + } + + // Simulate entry slippage (10% use MARKET fallback with slippage) + let entryPrice = currentEvent.price; + if (Math.random() < MARKET_FALLBACK_RATE) { + // MARKET order fallback - apply entry slippage + const isLong = side === 'SELL'; + entryPrice = isLong + ? entryPrice * (1 + EXIT_SLIPPAGE.ENTRY_MARKET) // BUY market: fills higher + : entryPrice * (1 - EXIT_SLIPPAGE.ENTRY_MARKET); // SELL market: fills lower + } + + const isLong = side === 'SELL'; // Buy on SELL liquidations, Sell on BUY liquidations + + const tpPrice = isLong + ? entryPrice * (1 + tpPercent/100) + : entryPrice * (1 - tpPercent/100); + const slPrice = isLong + ? entryPrice * (1 - slPercent/100) + : entryPrice * (1 + slPercent/100); + + activePositions.push({ + entryPrice, + entryTime: currentTime, + tpPrice, + slPrice, + isLong, + size: tradeSize * leverage / entryPrice + }); + lastEntryTime = currentTime; + lastHunterEntryTime = currentTime; + } + } + + while (priceIndex < allPrices.length) { + const priceBar = allPrices[priceIndex]; + evaluatePositionsOnBar(priceBar, priceIndex); + priceIndex++; + } + + if (activePositions.length > 0) { + const fallbackEvent = liquidations[liquidations.length - 1]; + const lastBar = allPrices.length > 0 + ? allPrices[allPrices.length - 1] + : { + event_time: fallbackEvent?.event_time || Date.now(), + price: fallbackEvent?.price ?? 0 + }; + + activePositions.forEach(pos => { + const fallbackPrice = typeof lastBar.price === 'number' && lastBar.price > 0 + ? lastBar.price + : pos.entryPrice; + recordExit(pos, fallbackPrice, 'EOD', lastBar.event_time || pos.entryTime); + }); + + activePositions = []; + } + + // Calculate statistics + const wins = completedTrades.filter(t => t.pnl > 0).length; + const losses = completedTrades.filter(t => t.pnl < 0).length; + const winRate = completedTrades.length > 0 ? (wins / completedTrades.length * 100) : 0; + const avgWin = wins > 0 ? completedTrades.filter(t => t.pnl > 0).reduce((sum, t) => sum + t.pnl, 0) / wins : 0; + const avgLoss = losses > 0 ? completedTrades.filter(t => t.pnl < 0).reduce((sum, t) => sum + t.pnl, 0) / losses : 0; + const avgDuration = completedTrades.length > 0 ? completedTrades.reduce((sum, t) => sum + t.duration, 0) / completedTrades.length / 1000 / 60 : 0; // minutes + + // Calculate risk metrics + const riskMetrics = calculateRiskMetrics(completedTrades); + + return { + totalTrades: completedTrades.length, + wins, + losses, + totalPnl, + winRate, + avgWin, + avgLoss, + avgDuration, + activePositions: activePositions.length, + recentTrades: completedTrades.slice(-3), + ...riskMetrics + }; +} + +// 4. Generate REALISTIC Backtest Optimization Recommendations +async function generateRecommendations(deployableCapital) { + console.log('REALISTIC BACKTEST OPTIMIZATION'); + console.log('===================================\n'); + + const recommendations = []; + const optimizedConfig = JSON.parse(JSON.stringify(config)); + const sanitizedCapital = Number.isFinite(deployableCapital) && deployableCapital > 0 ? deployableCapital : 0; + + const symbolEntries = Object.entries(config.symbols); + if (symbolEntries.length === 0) { + return { recommendations, optimizedConfig, recommendedGlobalMax: 0 }; + } + + const baselineTotalMargin = symbolEntries.reduce((sum, [, cfg]) => { + const baseMargin = cfg.maxPositionMarginUSDT || (cfg.tradeSize || 20) * 5; + const perSide = Number.isFinite(baseMargin) && baseMargin > 0 ? baseMargin : 0; + return sum + perSide * 2; + }, 0); + + const scaleFactor = baselineTotalMargin > 0 && sanitizedCapital > 0 + ? Math.max(0.25, Math.min(2.5, sanitizedCapital / baselineTotalMargin)) + : 1; + + for (const [symbol, symbolConfig] of symbolEntries) { + const spanDays = getSymbolDataSpanDays(symbol); + const fallbackMargin = (symbolConfig.tradeSize || 20) * 5; + const baseMargin = symbolConfig.maxPositionMarginUSDT || fallbackMargin; + const capitalBudget = Math.max(5, Math.min(sanitizedCapital || baseMargin, baseMargin * scaleFactor)); + + const optimization = await optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spanDays); + + const currentDaily = optimization.current.performance.dailyPnl; + const optimizedDaily = optimization.optimized.dailyPnl; + const delta = optimization.improvements.totalDaily; + + console.log('SUMMARY ' + symbol + ': current $' + currentDaily.toFixed(2) + + ' -> $' + optimizedDaily.toFixed(2) + ' (?? $' + delta.toFixed(2) + '/day)'); + + recommendations.push({ + symbol, + currentLong: optimization.current.longThreshold, + currentShort: optimization.current.shortThreshold, + optimizedLong: optimization.optimized.config.longVolumeThresholdUSDT, + optimizedShort: optimization.optimized.config.shortVolumeThresholdUSDT, + currentTradeSize: optimization.current.tradeSize, + currentLongTradeSize: optimization.current.longTradeSize, + currentShortTradeSize: optimization.current.shortTradeSize, + currentMargin: optimization.current.margin, + currentLeverage: optimization.current.leverage, + currentTp: optimization.current.tp, + currentSl: optimization.current.sl, + currentLongMaxPositions: optimization.current.longMaxPositions, + currentShortMaxPositions: optimization.current.shortMaxPositions, + currentTimeWindow: optimization.current.thresholdTimeWindow, + currentCooldown: optimization.current.thresholdCooldown, + optimizedTradeSize: optimization.optimized.config.tradeSize, + optimizedLongTradeSize: optimization.optimized.config.longTradeSize, + optimizedShortTradeSize: optimization.optimized.config.shortTradeSize, + optimizedMargin: optimization.optimized.config.maxPositionMarginUSDT, + optimizedLeverage: optimization.optimized.leverage, + optimizedTp: optimization.optimized.tp, + optimizedSl: optimization.optimized.sl, + optimizedLongMaxPositions: optimization.optimized.long.maxPositions, + optimizedShortMaxPositions: optimization.optimized.short.maxPositions, + optimizedTimeWindow: optimization.optimized.config.thresholdTimeWindow, + optimizedCooldown: optimization.optimized.config.thresholdCooldown, + longImprovement: optimization.improvements.long, + shortImprovement: optimization.improvements.short, + totalDailyImprovement: optimization.improvements.totalDaily, + currentPerformance: optimization.current.performance, + optimizedPerformance: { + long: optimization.optimized.long.result, + short: optimization.optimized.short.result, + totalPnl: optimization.optimized.totalPnl, + dailyPnl: optimization.optimized.dailyPnl + }, + optimizedScore: optimization.optimized.finalScore, + optimizedSharpe: optimization.optimized.sharpeRatio, + optimizedDrawdownScore: optimization.optimized.drawdownScore, + optimizedConfig: optimization.optimized.config, + spanDays: optimization.spanDays + }); + + // CRITICAL FIX: Apply the optimized config to the final optimizedConfig object + // This was missing - we were collecting recommendations but never updating optimizedConfig.symbols! + optimizedConfig.symbols[symbol] = { + ...optimizedConfig.symbols[symbol], + ...optimization.optimized.config + }; + } + + console.log('KEY BACKTEST INSIGHTS:'); + console.log('- Optimization considers thresholds, TP/SL, trade size, leverage, and margin per symbol'); + console.log('- Deployable capital scaled to $' + formatLargeNumber(sanitizedCapital)); + console.log('- VWAP protection disabled automatically where aggressive thresholds outperform'); + console.log(); + + // In hedge mode, maxOpenPositions counts unique symbols (hedged pairs count as one) + // Each symbol can have LONG + SHORT positions, but counts as 1 for the global limit + const recommendedGlobalMax = recommendations.length; + + const recommendedGlobalRounded = Math.max(1, Math.ceil(recommendedGlobalMax)); + + if (!optimizedConfig.global) { + optimizedConfig.global = {}; + } + optimizedConfig.global.maxOpenPositions = recommendedGlobalRounded; + + const currentGlobalCap = config.global?.maxOpenPositions ?? 'n/a'; + console.log('Recommended global max open positions: ' + recommendedGlobalRounded + ' (current ' + currentGlobalCap + ')'); + console.log(); + + return { recommendations, optimizedConfig, recommendedGlobalMax: recommendedGlobalRounded }; +}// Capital allocation analysis for liquidation cascade strategy +function analyzeCapitalAllocation(balance, accountInfo, positions) { + console.log('???? COMPLETE ACCOUNT SNAPSHOT'); + console.log('=============================\n'); + + // Debug: Show raw account info + console.log('???? DEBUG - Raw Account Data:'); + console.log('Balance API Response:', JSON.stringify(balance, null, 2)); + if (accountInfo) { + console.log('Account API Response keys:', Object.keys(accountInfo)); + console.log('Account totalMarginBalance:', accountInfo.totalMarginBalance); + console.log('Account totalWalletBalance:', accountInfo.totalWalletBalance); + } + console.log(); + + console.log(`???? Account Balance Breakdown:`); + console.log(` Available Balance: $${formatLargeNumber(balance.availableBalance)}`); + + if (accountInfo) { + // Calculate total account value properly + const totalMarginBalance = parseFloat(accountInfo.totalMarginBalance || 0); + const totalWalletBalance = parseFloat(accountInfo.totalWalletBalance || 0); + const totalUnrealizedPNL = parseFloat(accountInfo.totalUnrealizedProfit || 0); + const usedMargin = parseFloat(accountInfo.totalInitialMargin || 0); + + console.log(` Total Margin Balance: $${formatLargeNumber(totalMarginBalance)}`); + console.log(` Total Wallet Balance: $${formatLargeNumber(totalWalletBalance)}`); + console.log(` Used in Positions: $${formatLargeNumber(usedMargin)}`); + console.log(` Unrealized PNL: $${formatLargeNumber(totalUnrealizedPNL)}`); + console.log(` Maintenance Margin: $${formatLargeNumber(parseFloat(accountInfo.totalMaintMargin || 0))}`); + + // Calculate true total account value + const trueTotal = Math.max(totalMarginBalance, totalWalletBalance, balance.availableBalance + usedMargin); + console.log(` ???? CALCULATED TOTAL: $${formatLargeNumber(trueTotal)}`); + + // Store for later use + this.calculatedTotal = trueTotal; + } + console.log(); + + if (positions && positions.length > 0) { + console.log(`???? CURRENT POSITIONS (${positions.length} active):`); + console.log(`Symbol | Side | Size | Entry Price | Mark Price | PNL | Margin | ROE%`); + console.log(`------------|------|-----------|-------------|-------------|----------|----------|------`); + + let totalPNL = 0; + let totalMargin = 0; + + positions.forEach(pos => { + const pnl = parseFloat(pos.unRealizedProfit || 0); + const positionAmt = parseFloat(pos.positionAmt || 0); + const leverage = parseFloat(pos.leverage || 1) || 1; + const entryPrice = parseFloat(pos.entryPrice || 0); + const markPrice = parseFloat(pos.markPrice || 0); + const notional = Math.abs(positionAmt * entryPrice); + const marginFromExchange = parseFloat(pos.initialMargin || pos.positionInitialMargin || pos.isolatedMargin || 0); + const derivedMargin = leverage > 0 ? notional / leverage : 0; + const margin = marginFromExchange > 0 ? marginFromExchange : derivedMargin; + const roe = margin > 0 ? (pnl / margin) * 100 : 0; + + totalPNL += pnl; + totalMargin += margin; + + const side = positionAmt > 0 ? 'LONG' : 'SHORT'; + const size = Math.abs(positionAmt); + + console.log(`${pos.symbol.padEnd(11)} | ${side.padEnd(4)} | ${size.toFixed(4).padEnd(9)} | $${entryPrice.toFixed(4).padEnd(10)} | $${markPrice.toFixed(4).padEnd(10)} | $${pnl.toFixed(2).padEnd(8)} | $${margin.toFixed(2).padEnd(8)} | ${roe.toFixed(1)}%`); + }); + + console.log(`------------|------|-----------|-------------|-------------|----------|----------|------`); + console.log(`TOTALS | | | | | $${totalPNL.toFixed(2).padEnd(8)} | $${totalMargin.toFixed(2).padEnd(8)} |`); + console.log(); + } + + let totalMaxAllocation = 0; + const globalSettings = config.global; + + console.log(`?????? Global Settings:`); + console.log(` Risk Percent: ${globalSettings.riskPercent}%`); + console.log(` Max Open Positions: ${globalSettings.maxOpenPositions}`); + console.log(` Position Mode: ${globalSettings.positionMode}`); + console.log(); + + console.log(`???? Per-Symbol Capital Allocation:`); + console.log(`Symbol | Trade Size | Max Margin/Side | Max Positions | Strategy`); + console.log(`------------|------------|------------------|---------------|----------`); + + for (const [symbol, symbolConfig] of Object.entries(config.symbols)) { + const tradeSize = symbolConfig.tradeSize || 20; + const shortTradeSize = symbolConfig.shortTradeSize || tradeSize; + const maxMarginPerSide = symbolConfig.maxPositionMarginUSDT || 100; + const maxLongPositions = Math.floor(maxMarginPerSide / tradeSize); + const maxShortPositions = Math.floor(maxMarginPerSide / shortTradeSize); + + totalMaxAllocation += maxMarginPerSide * 2; + + const _longThreshold = symbolConfig.longVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + const _shortThreshold = symbolConfig.shortVolumeThresholdUSDT || symbolConfig.volumeThresholdUSDT || 0; + + console.log(`${symbol.padEnd(11)} | $${tradeSize.toString().padEnd(9)} | $${maxMarginPerSide.toString().padEnd(16)} | ${maxLongPositions}L/${maxShortPositions}S${' '.padEnd(8)} | Cascade`); + } + + console.log(`------------|------------|------------------|---------------|----------`); + console.log(`TOTAL | | $${totalMaxAllocation.toString().padEnd(16)} | |`); + console.log(); + + const utilizationRate = (totalMaxAllocation / balance.availableBalance * 100).toFixed(1); + const safeUtilization = utilizationRate <= 80 ? '???' : utilizationRate <= 95 ? '??????' : '????'; + + console.log(`???? Capital Utilization:`); + console.log(` Max Allocation (both sides): $${formatLargeNumber(totalMaxAllocation)} (${utilizationRate}% of available) ${safeUtilization}`); + console.log(` Safe Range: ???80% optimal, ???95% acceptable`); + console.log(); + + return { totalMaxAllocation, utilizationRate, calculatedTotal: this.calculatedTotal || 0 }; +} + +// Enhanced liquidation cascade analysis +function analyzeLiquidationCascades() { + console.log('???? LIQUIDATION CASCADE ANALYSIS'); + console.log('================================\n'); + + console.log('Strategy: Multiple Position Accumulation During Cascades'); + console.log('- Liquidation cascades create price dislocations'); + console.log('- Accumulate positions as price moves against liquidated traders'); + console.log('- Average down during cascades, exit on rebound\n'); + + // Analyze cascade patterns in the data + const cascadeAnalysis = db.prepare(` + SELECT + symbol, + datetime(event_time/1000, 'unixepoch', 'start of minute') as minute, + COUNT(*) as liquidation_count, + SUM(volume_usdt) as total_volume, + side + FROM liquidations + GROUP BY symbol, minute, side + HAVING liquidation_count >= 3 OR total_volume >= 10000 + ORDER BY symbol, minute + `).all(); + + const cascadesBySymbol = {}; + cascadeAnalysis.forEach(row => { + if (!cascadesBySymbol[row.symbol]) cascadesBySymbol[row.symbol] = []; + cascadesBySymbol[row.symbol].push(row); + }); + + console.log('???? Cascade Frequency by Symbol:'); + console.log('Symbol | Cascade Minutes | Avg Volume/Min | Max Liquidations/Min'); + console.log('------------|----------------|----------------|--------------------'); + + for (const [symbol, cascades] of Object.entries(cascadesBySymbol)) { + if (!config.symbols[symbol]) continue; + + const avgVolume = cascades.reduce((sum, c) => sum + c.total_volume, 0) / cascades.length; + const maxLiqs = Math.max(...cascades.map(c => c.liquidation_count)); + + console.log(`${symbol.padEnd(11)} | ${cascades.length.toString().padEnd(14)} | $${formatLargeNumber(avgVolume).padEnd(13)} | ${maxLiqs}`); + } + console.log(); +} + +// Main execution - now async +async function analyzeRealTradingHistory(credentials) { + console.log('???? REAL TRADING HISTORY ANALYSIS'); + console.log('=================================\n'); + + const symbols = ['ASTERUSDT']; + const sevenDaysAgo = Date.now() - (7 * 24 * 60 * 60 * 1000); + + for (const symbol of symbols) { + if (!config.symbols[symbol]) continue; + + console.log(`???? ${symbol} Real Trade Analysis:`); + + try { + // Get trades from last 7 days + const trades = await getUserTrades(credentials, symbol, 1000, sevenDaysAgo); + + if (trades.length === 0) { + console.log(` No trades found in last 7 days`); + console.log(); + continue; + } + + // Analyze real trading performance + let totalPnl = 0; + let wins = 0; + let losses = 0; + let totalCommission = 0; + + trades.forEach(trade => { + const pnl = parseFloat(trade.realizedPnl); + const commission = parseFloat(trade.commission); + + totalPnl += pnl; + totalCommission += Math.abs(commission); + + if (pnl > 0) wins++; + else if (pnl < 0) losses++; + }); + + const totalTrades = wins + losses; + const winRate = totalTrades > 0 ? (wins / totalTrades) * 100 : 0; + const avgWin = wins > 0 ? trades.filter(t => parseFloat(t.realizedPnl) > 0).reduce((sum, t) => sum + parseFloat(t.realizedPnl), 0) / wins : 0; + const avgLoss = losses > 0 ? Math.abs(trades.filter(t => parseFloat(t.realizedPnl) < 0).reduce((sum, t) => sum + parseFloat(t.realizedPnl), 0) / losses) : 0; + const dailyPnl = totalPnl / 7; + + console.log(` ???? Real Performance (Last 7 days):`); + console.log(` Total Trades: ${trades.length} fills (${totalTrades} with PnL)`); + console.log(` Win Rate: ${winRate.toFixed(1)}% (${wins}W/${losses}L)`); + console.log(` Total PnL: $${totalPnl.toFixed(2)} | Daily: $${dailyPnl.toFixed(2)}/day`); + console.log(` Avg Win: $${avgWin.toFixed(2)} | Avg Loss: $${avgLoss.toFixed(2)}`); + console.log(` Total Commissions: $${totalCommission.toFixed(2)}`); + + // Compare with backtest + const symbolConfig = config.symbols[symbol]; + const currentLong = symbolConfig.longVolumeThresholdUSDT || 0; + const tradeSize = symbolConfig.tradeSize || 20; + const leverage = symbolConfig.leverage || 10; + const tpPercent = symbolConfig.tpPercent || 1; + const slPercent = symbolConfig.slPercent || 5; + const maxPositions = Math.floor((symbolConfig.maxPositionMarginUSDT || 100) / tradeSize); + + const cooldownMs = symbolConfig.thresholdCooldown || DEFAULT_THRESHOLD_COOLDOWN_MS; + const timeWindowMs = symbolConfig.thresholdTimeWindow || DEFAULT_THRESHOLD_WINDOW_MS; + const backtestResult = await backtestSymbol(symbol, 'SELL', currentLong, maxPositions, tradeSize, leverage, tpPercent, slPercent, { + cooldownMs, + hunterCooldownMs: HUNTER_COOLDOWN_MS, + windowMs: timeWindowMs + }); + const spanDays = getSymbolDataSpanDays(symbol); + const backtestDaily = spanDays > 0 ? backtestResult.totalPnl / spanDays : 0; + + console.log(` ???? Backtest vs Reality Comparison:`); + console.log(` Backtest Daily: $${backtestDaily.toFixed(2)}/day | Real Daily: $${dailyPnl.toFixed(2)}/day`); + console.log(` Backtest Win Rate: ${backtestResult.winRate.toFixed(1)}% | Real Win Rate: ${winRate.toFixed(1)}%`); + console.log(` Accuracy: ${Math.abs(backtestResult.winRate - winRate) < 10 ? '??? Good' : '??? Poor'} (${Math.abs(backtestResult.winRate - winRate).toFixed(1)}% diff)`); + + if (Math.abs(dailyPnl - backtestDaily) > 50) { + console.log(` ?????? Large discrepancy detected - backtest may need refinement`); + } + + } catch (error) { + console.log(` ??? Failed to analyze ${symbol}: ${error.message}`); + } + + console.log(); + } +} + +// Capital allocation optimizer +function optimizeCapitalAllocation(accountInfo, recommendations, symbolConfigs = config.symbols) { + console.log('???? CAPITAL ALLOCATION OPTIMIZER'); + console.log('================================\n'); + + const totalWalletBalance = parseFloat(accountInfo?.totalWalletBalance ?? 0); + const availableBalance = parseFloat(accountInfo?.availableBalance ?? 0); + const targetUtilization = 0.80; // 80% max utilization + const maxAllocation = totalWalletBalance * targetUtilization; + + console.log(`???? Total Wallet Balance: $${formatLargeNumber(totalWalletBalance)}`); + console.log(`???? Available Balance: $${formatLargeNumber(availableBalance)}`); + console.log(`???? Target Utilization: ${(targetUtilization * 100).toFixed(0)}%`); + console.log(`???? Max Safe Allocation: $${formatLargeNumber(maxAllocation)}\n`); + + // Calculate total required allocation with current config + let currentTotalAllocation = 0; + for (const [_symbol, symbolConfig] of Object.entries(symbolConfigs)) { + currentTotalAllocation += symbolConfig.maxPositionMarginUSDT || 100; + } + + console.log(`???? Current Total Allocation: $${formatLargeNumber(currentTotalAllocation)} (${(currentTotalAllocation / totalWalletBalance * 100).toFixed(1)}% of total balance)`); + + if (currentTotalAllocation > maxAllocation) { + console.log(`?????? OVERALLOCATED by $${formatLargeNumber(currentTotalAllocation - maxAllocation)}`); + console.log(`???? Recommendation: Reduce per-symbol allocation or disable low-performing symbols\n`); + } else { + console.log(`??? Capital allocation within safe range\n`); + } + + // Rank symbols by expected daily profit improvement + const rankedSymbols = recommendations + .sort((a, b) => b.totalDailyImprovement - a.totalDailyImprovement); + + console.log('???? Symbol Priority by Expected Daily Profit Improvement:'); + console.log('Rank | Symbol | Current Daily | Optimized Daily | Improvement | Allocation'); + console.log('-----|-----------|---------------|-----------------|-------------|------------'); + + rankedSymbols.forEach((_rec, i) => { + const symbolConfig = symbolConfigs[_rec.symbol]; + const allocation = symbolConfig.maxPositionMarginUSDT || 100; + const currentDaily = _rec.currentPerformance.dailyPnl; + const optimizedDaily = currentDaily + _rec.totalDailyImprovement; + const improvement = _rec.totalDailyImprovement; + + console.log( + `${(i + 1).toString().padEnd(4)} | ` + + `${_rec.symbol.padEnd(9)} | ` + + `$${currentDaily.toFixed(2).padEnd(13)} | ` + + `$${optimizedDaily.toFixed(2).padEnd(15)} | ` + + `+$${improvement.toFixed(2).padEnd(11)} | ` + + `$${formatLargeNumber(allocation)}` + ); + }); + + console.log(); + + // Suggest allocation rebalancing + if (currentTotalAllocation > maxAllocation) { + console.log('???? REBALANCING RECOMMENDATIONS:'); + const allocationPerSymbol = Math.floor(maxAllocation / Object.keys(config.symbols).length); + + console.log(` Option 1: Equal allocation of $${formatLargeNumber(allocationPerSymbol)} per symbol`); + console.log(` Option 2: Weighted by expected profitability (top performers get more)`); + console.log(` Option 3: Disable bottom 25% performers and reallocate to top performers\n`); + } + + return { + currentAllocation: currentTotalAllocation, + maxSafeAllocation: maxAllocation, + isOverallocated: currentTotalAllocation > maxAllocation, + rankedSymbols + }; +} + +// Generate optimization summary with actionable recommendations +function generateOptimizationSummary(recommendations, capitalOptimization, optimizedConfig, recommendedGlobalMax) { + console.log('???? OPTIMIZATION SUMMARY'); + console.log('======================\n'); + + // Calculate total improvements + const totalDailyImprovement = recommendations.reduce((sum, _rec) => sum + _rec.totalDailyImprovement, 0); + const totalCurrentDaily = recommendations.reduce((sum, _rec) => sum + _rec.currentPerformance.dailyPnl, 0); + const totalOptimizedDaily = totalCurrentDaily + totalDailyImprovement; + const improvementPercent = Math.abs(totalCurrentDaily) > 1e-6 + ? (totalDailyImprovement / Math.abs(totalCurrentDaily)) * 100 + : null; + + console.log('???? PERFORMANCE SUMMARY:'); + console.log(` Current Daily P&L: $${totalCurrentDaily.toFixed(2)}`); + console.log(` Optimized Daily P&L: $${totalOptimizedDaily.toFixed(2)}`); + const improvementText = improvementPercent === null + ? 'n/a (baseline ??? 0)' + : `${improvementPercent.toFixed(1)}%`; + console.log(` Total Daily Improvement: +$${totalDailyImprovement.toFixed(2)} (+${improvementText})`); + console.log(` Monthly Improvement: +$${(totalDailyImprovement * 30).toFixed(2)}\n`); + + // Threshold recommendations + console.log('???? RECOMMENDED THRESHOLD CHANGES:'); + recommendations.forEach(_rec => { + if (_rec.optimizedLong !== _rec.currentLong || _rec.optimizedShort !== _rec.currentShort) { + console.log(` ${_rec.symbol}:`); + if (_rec.optimizedLong !== _rec.currentLong) { + console.log(` LONG: $${formatLargeNumber(_rec.currentLong)} ??? $${formatLargeNumber(_rec.optimizedLong)} (+$${_rec.longImprovement.toFixed(2)}/day)`); + } + if (_rec.optimizedShort !== _rec.currentShort) { + console.log(` SHORT: $${formatLargeNumber(_rec.currentShort)} ??? $${formatLargeNumber(_rec.optimizedShort)} (+$${_rec.shortImprovement.toFixed(2)}/day)`); + } + console.log(` Trade Size (L/S): $${_rec.currentLongTradeSize.toFixed(2)} ??? $${_rec.optimizedLongTradeSize.toFixed(2)} / $${_rec.currentShortTradeSize.toFixed(2)} ??? $${_rec.optimizedShortTradeSize.toFixed(2)}`); + console.log(` TP/SL: ${_rec.currentTp.toFixed(2)}%/${_rec.currentSl.toFixed(2)}% ??? ${_rec.optimizedTp.toFixed(2)}%/${_rec.optimizedSl.toFixed(2)}%`); + console.log(` Leverage: ${_rec.currentLeverage.toFixed(2)}x ??? ${_rec.optimizedLeverage.toFixed(2)}x`); + if (_rec.optimizedScore !== undefined) { + console.log(` Score: ${_rec.optimizedScore.toFixed(2)} (PnL: ${formatWeightPercent(scoringWeights.percent.pnl)}, Sharpe: ${formatWeightPercent(scoringWeights.percent.sharpe)}, Drawdown: ${formatWeightPercent(scoringWeights.percent.drawdown)})`); + } + } + }); + + console.log(); + + // Capital allocation recommendations + if (capitalOptimization.isOverallocated) { + console.log('?????? CAPITAL ALLOCATION WARNING:'); + console.log(` Current: $${formatLargeNumber(capitalOptimization.currentAllocation)}`); + console.log(` Safe Max: $${formatLargeNumber(capitalOptimization.maxSafeAllocation)}`); + console.log(` Overallocated by: $${formatLargeNumber(capitalOptimization.currentAllocation - capitalOptimization.maxSafeAllocation)}\n`); + } + + if (recommendedGlobalMax) { + const currentGlobal = config.global?.maxOpenPositions; + console.log('???? Global Position Capacity:'); + console.log(` Current maxOpenPositions: ${currentGlobal ?? 'n/a'}`); + console.log(` Recommended maxOpenPositions: ${recommendedGlobalMax}`); + console.log(); + } + + // Export recommendations as JSON + const exportData = { + timestamp: new Date().toISOString(), + summary: { + currentDailyPnl: totalCurrentDaily, + optimizedDailyPnl: totalOptimizedDaily, + dailyImprovement: totalDailyImprovement, + monthlyImprovement: totalDailyImprovement * 30, + improvementPercent: improvementPercent, + recommendedMaxOpenPositions: recommendedGlobalMax + }, + recommendations: recommendations.map(_rec => ({ + symbol: _rec.symbol, + thresholds: { + current: { long: _rec.currentLong, short: _rec.currentShort }, + optimized: { long: _rec.optimizedLong, short: _rec.optimizedShort } + }, + settings: { + current: { + tradeSize: _rec.currentTradeSize, + longTradeSize: _rec.currentLongTradeSize, + shortTradeSize: _rec.currentShortTradeSize, + maxPositionMarginUSDT: _rec.currentMargin, + leverage: _rec.currentLeverage, + tpPercent: _rec.currentTp, + slPercent: _rec.currentSl, + maxPositionsLong: _rec.currentLongMaxPositions, + maxPositionsShort: _rec.currentShortMaxPositions, + thresholdTimeWindow: _rec.currentTimeWindow, + thresholdCooldown: _rec.currentCooldown + }, + optimized: { + tradeSize: _rec.optimizedTradeSize, + longTradeSize: _rec.optimizedLongTradeSize, + shortTradeSize: _rec.optimizedShortTradeSize, + maxPositionMarginUSDT: _rec.optimizedMargin, + leverage: _rec.optimizedLeverage, + tpPercent: _rec.optimizedTp, + slPercent: _rec.optimizedSl, + maxPositionsLong: _rec.optimizedLongMaxPositions, + maxPositionsShort: _rec.optimizedShortMaxPositions, + thresholdTimeWindow: _rec.optimizedTimeWindow, + thresholdCooldown: _rec.optimizedCooldown, + vwapProtection: _rec.optimizedConfig?.vwapProtection + } + }, + improvement: { + long: _rec.longImprovement, + short: _rec.shortImprovement, + total: _rec.totalDailyImprovement + }, + performance: { + current: { + long: { + trades: _rec.currentPerformance.long.totalTrades, + winRate: _rec.currentPerformance.long.winRate, + pnl: _rec.currentPerformance.long.totalPnl, + sharpe: _rec.currentPerformance.long.sharpeRatio, + maxDrawdown: _rec.currentPerformance.long.maxDrawdown + }, + short: { + trades: _rec.currentPerformance.short.totalTrades, + winRate: _rec.currentPerformance.short.winRate, + pnl: _rec.currentPerformance.short.totalPnl, + sharpe: _rec.currentPerformance.short.sharpeRatio, + maxDrawdown: _rec.currentPerformance.short.maxDrawdown + } + }, + optimized: { + long: { + trades: _rec.optimizedPerformance.long.totalTrades, + winRate: _rec.optimizedPerformance.long.winRate, + pnl: _rec.optimizedPerformance.long.totalPnl, + sharpe: _rec.optimizedPerformance.long.sharpeRatio, + maxDrawdown: _rec.optimizedPerformance.long.maxDrawdown + }, + short: { + trades: _rec.optimizedPerformance.short.totalTrades, + winRate: _rec.optimizedPerformance.short.winRate, + pnl: _rec.optimizedPerformance.short.totalPnl, + sharpe: _rec.optimizedPerformance.short.sharpeRatio, + maxDrawdown: _rec.optimizedPerformance.short.maxDrawdown + } + } + }, + scoring: { + finalScore: _rec.optimizedScore, + sharpeRatio: _rec.optimizedSharpe, + drawdownScore: _rec.optimizedDrawdownScore, + weights: { + pnl: normalizedScoringWeights.pnl, + sharpe: normalizedScoringWeights.sharpe, + drawdown: normalizedScoringWeights.drawdown + }, + weightPercent: { + pnl: scoringWeights.percent.pnl, + sharpe: scoringWeights.percent.sharpe, + drawdown: scoringWeights.percent.drawdown + } + } + })), + capitalAllocation: capitalOptimization, + recommendedMaxOpenPositions: recommendedGlobalMax, + optimizedConfig + }; + + // Write to JSON file + fs.writeFileSync( + path.join(__dirname, 'optimization-results.json'), + JSON.stringify(exportData, null, 2) + ); + + console.log('???? Results saved to: optimization-results.json\n'); + + return exportData; +} + +function askYesNo(question) { + return new Promise(resolve => { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + rl.question(question, answer => { + rl.close(); + const normalized = answer.trim().toLowerCase(); + resolve(normalized === 'y' || normalized === 'yes'); + }); + }); +} + +async function maybeApplyOptimizedConfig(originalConfig, optimizedConfig, summary) { + const autoMode = FORCE_OPTIMIZER_OVERWRITE; + const autoConfirm = FORCE_OPTIMIZER_CONFIRM || autoMode; + + const canPrompt = Boolean(process.stdin.isTTY || process.stdout.isTTY); + if (!canPrompt && !autoMode) { + console.log('dY>` No interactive TTY detected. Skipping config overwrite prompt (set FORCE_OPTIMIZER_OVERWRITE or run in an interactive shell).'); + return; + } + + console.log('dY"S Optimization Delta Overview:'); + if (summary) { + console.log(` Current Daily P&L: ${formatCurrency(summary.currentDailyPnl)}`); + console.log(` Optimized Daily P&L: ${formatCurrency(summary.optimizedDailyPnl)}`); + console.log(` Daily Improvement: ${formatCurrency(summary.dailyImprovement)} (Monthly +${formatCurrency(summary.monthlyImprovement)})`); + if (summary.recommendedMaxOpenPositions) { + console.log(` Current maxOpenPositions: ${config.global?.maxOpenPositions ?? 'n/a'}`); + console.log(` Recommended maxOpenPositions: ${summary.recommendedMaxOpenPositions}`); + } + } else { + console.log(' (Detailed summary unavailable)'); + } + console.log(); + + let confirm = autoConfirm; + if (!confirm) { + confirm = await askYesNo('Overwrite config.user.json with optimized settings? (y/N): '); + } else { + console.log('dY"? Auto-confirm enabled via FORCE_OPTIMIZER_OVERWRITE/CONFIRM environment variables'); + } + + if (!confirm) { + console.log('???sT???,? Keeping existing config.user.json'); + return; + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const backupPath = path.join(__dirname, `config.user.backup-${timestamp}.json`); + + fs.writeFileSync(backupPath, JSON.stringify(originalConfig, null, 2)); + fs.writeFileSync(configPath, JSON.stringify(optimizedConfig, null, 2)); + + console.log(`dY-, Backup saved to ${backupPath}`); + console.log('???o. config.user.json overwritten with optimized settings'); +} + +async function main() { + try { + const weightSummary = `${formatWeightPercent(scoringWeights.percent.pnl)} / ${formatWeightPercent(scoringWeights.percent.sharpe)} / ${formatWeightPercent(scoringWeights.percent.drawdown)}`; + const weightLabel = scoringWeights.isDefault ? ' (default)' : ''; + console.log(`???? Using scoring weights (PnL / Sharpe / Drawdown): ${weightSummary}${weightLabel}\n`); + + console.log('???? Fetching complete account snapshot...\n'); + const [balance, accountInfo, positions] = await Promise.all([ + getAccountBalance(config.api), + getAccountInfo(config.api), + getCurrentPositions(config.api) + ]); + + // Core analyses + const _avgGap = analyzePriceDataCoverage(); + const capitalInfo = analyzeCapitalAllocation(balance, accountInfo, positions); + analyzeLiquidationCascades(); + analyzeCurrentConfig(); + await analyzeRealTradingHistory(config.api); + optimizeThresholds(); + rankSymbolProfitability(); + + // Generate recommendations with backtest + // Use CALCULATED TOTAL for optimal capital allocation + const deployableCapital = capitalInfo.calculatedTotal || parseFloat(accountInfo?.totalWalletBalance ?? 0); + const { recommendations, optimizedConfig, recommendedGlobalMax } = await generateRecommendations(deployableCapital); + + // Optimize capital allocation + const capitalOptimization = optimizeCapitalAllocation(accountInfo, recommendations, optimizedConfig.symbols); + + // Generate final summary + const optimizationResults = generateOptimizationSummary(recommendations, capitalOptimization, optimizedConfig, recommendedGlobalMax); + + await maybeApplyOptimizedConfig(config, optimizedConfig, optimizationResults.summary); + + console.log('???? Optimization analysis complete!'); + const totalValue = parseFloat(accountInfo?.totalMarginBalance || balance.totalWalletBalance || 0); + console.log(`???? Total account value: $${formatLargeNumber(totalValue)}`); + console.log('???? Strategy: Accumulate positions during cascades, profit on rebounds'); + console.log('\n???? Review optimization-results.json for detailed recommendations and the fully optimized config snapshot'); + + } catch (error) { + console.error('??? Error:', error.message); + console.error(error.stack); + } finally { + db.close(); + } +} + +// Run the analysis +main(); + + + diff --git a/package-lock.json b/package-lock.json index 7ffdf30..0f4565f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "@types/sqlite3": "^5.1.0", "@types/uuid": "^11.0.0", "axios": "^1.12.2", + "better-sqlite3": "^12.4.1", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "ethers": "^6.15.0", @@ -35,6 +36,7 @@ "react": "19.1.0", "react-dom": "19.1.0", "recharts": "^2.15.4", + "server-only": "^0.0.1", "sonner": "^2.0.7", "sqlite3": "^5.1.7", "tailwind-merge": "^3.3.1", @@ -5506,6 +5508,20 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/better-sqlite3": { + "version": "12.4.1", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.4.1.tgz", + "integrity": "sha512-3yVdyZhklTiNrtg+4WqHpJpFDd+WHTg2oM7UcR80GqL05AOV0xEJzc6qNvFYoEtE+hRp1n9MpN6/+4yhlGkDXQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x" + } + }, "node_modules/bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", @@ -12250,6 +12266,12 @@ "node": ">=10" } }, + "node_modules/server-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz", + "integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==", + "license": "MIT" + }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", diff --git a/package.json b/package.json index 4d6e469..56d58e2 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "test:ws": "tsx tests/core/websocket.test.ts", "test:errors": "tsx tests/core/error-logging.test.ts", "test:integration": "tsx tests/integration/trading-flow.test.ts", - "test:watch": "tsx watch tests/**/*.test.ts" + "test:watch": "tsx watch tests/**/*.test.ts", + "optimize:ui": "node optimize-config.js" }, "dependencies": { "@radix-ui/react-dialog": "^1.1.15", @@ -43,6 +44,7 @@ "@types/sqlite3": "^5.1.0", "@types/uuid": "^11.0.0", "axios": "^1.12.2", + "better-sqlite3": "^12.4.1", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "ethers": "^6.15.0", @@ -54,6 +56,7 @@ "react": "19.1.0", "react-dom": "19.1.0", "recharts": "^2.15.4", + "server-only": "^0.0.1", "sonner": "^2.0.7", "sqlite3": "^5.1.7", "tailwind-merge": "^3.3.1", diff --git a/src/app/api/optimizer/apply/route.ts b/src/app/api/optimizer/apply/route.ts new file mode 100644 index 0000000..7f744db --- /dev/null +++ b/src/app/api/optimizer/apply/route.ts @@ -0,0 +1,62 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { withAuth } from '@/lib/auth/with-auth'; +import { applyOptimizedConfig } from '@/lib/services/optimizerService'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/optimizer/apply + * + * Apply the optimized configuration from a completed job + * Creates a backup before applying changes + * + * Request body: + * { + * jobId: "opt_123abc" + * } + * + * Response: + * { + * success: true, + * backupPath: "config.user.backup-2025-10-05T12-34-56.json", + * message: "Configuration applied successfully" + * } + */ +export const POST = withAuth(async (request: NextRequest) => { + try { + const body = await request.json(); + + if (!body.jobId) { + return NextResponse.json( + { success: false, error: 'Missing jobId' }, + { status: 400 } + ); + } + + const result = await applyOptimizedConfig(body.jobId); + + if (!result.success) { + return NextResponse.json( + { success: false, error: result.error }, + { status: 400 } + ); + } + + return NextResponse.json({ + success: true, + backupPath: result.backupPath, + message: 'Configuration applied successfully. Bot will reload config automatically.', + }); + } catch (error) { + console.error('Error applying optimized config:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Failed to apply configuration', + }, + { status: 500 } + ); + } +}); + diff --git a/src/app/api/optimizer/cancel/route.ts b/src/app/api/optimizer/cancel/route.ts new file mode 100644 index 0000000..3a30f03 --- /dev/null +++ b/src/app/api/optimizer/cancel/route.ts @@ -0,0 +1,59 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { withAuth } from '@/lib/auth/with-auth'; +import { cancelJob } from '@/lib/services/optimizerService'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/optimizer/cancel + * + * Cancel a running optimization job + * + * Request body: + * { + * jobId: "opt_123abc" + * } + * + * Response: + * { + * success: true, + * message: "Optimization cancelled" + * } + */ +export const POST = withAuth(async (request: NextRequest) => { + try { + const body = await request.json(); + + if (!body.jobId) { + return NextResponse.json( + { success: false, error: 'Missing jobId' }, + { status: 400 } + ); + } + + const cancelled = cancelJob(body.jobId); + + if (!cancelled) { + return NextResponse.json( + { success: false, error: 'Job not found or cannot be cancelled' }, + { status: 404 } + ); + } + + return NextResponse.json({ + success: true, + message: 'Optimization cancelled successfully', + }); + } catch (error) { + console.error('Error cancelling optimization:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Failed to cancel optimization', + }, + { status: 500 } + ); + } +}); + diff --git a/src/app/api/optimizer/reset/route.ts b/src/app/api/optimizer/reset/route.ts new file mode 100644 index 0000000..e440f8a --- /dev/null +++ b/src/app/api/optimizer/reset/route.ts @@ -0,0 +1,38 @@ +import { NextResponse } from 'next/server'; +import { withAuth } from '@/lib/auth/with-auth'; +import { resetOptimizerState } from '@/lib/services/optimizerService'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/optimizer/reset + * + * Reset all optimizer state - clears cached jobs and results files + * This allows starting fresh optimizations without server restart + * + * Response: + * { + * success: true, + * message: "Optimizer state reset successfully" + * } + */ +export const POST = withAuth(async () => { + try { + resetOptimizerState(); + + return NextResponse.json({ + success: true, + message: 'Optimizer state reset successfully', + }); + } catch (error) { + console.error('Error resetting optimizer state:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Failed to reset optimizer state', + }, + { status: 500 } + ); + } +}); diff --git a/src/app/api/optimizer/run/route.ts b/src/app/api/optimizer/run/route.ts new file mode 100644 index 0000000..a338999 --- /dev/null +++ b/src/app/api/optimizer/run/route.ts @@ -0,0 +1,107 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { withAuth } from '@/lib/auth/with-auth'; +import { startOptimization } from '@/lib/services/optimizerService'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/optimizer/run + * + * Start a new optimization job + * + * Request body: + * { + * weights: { pnl: 50, sharpe: 30, drawdown: 20 }, + * capitalAllocation?: number, + * symbols?: string[] + * } + * + * Response: + * { + * success: true, + * jobId: "opt_123abc", + * estimatedDuration: "10-30 minutes" + * } + */ +export const POST = withAuth(async (request: NextRequest) => { + try { + const body = await request.json(); + + // Validate request body + if (!body.weights) { + return NextResponse.json( + { success: false, error: 'Missing weights configuration' }, + { status: 400 } + ); + } + + const { pnl, sharpe, drawdown } = body.weights; + + // Validate weights + if ( + typeof pnl !== 'number' || + typeof sharpe !== 'number' || + typeof drawdown !== 'number' + ) { + return NextResponse.json( + { success: false, error: 'Weights must be numbers' }, + { status: 400 } + ); + } + + const totalWeight = pnl + sharpe + drawdown; + if (Math.abs(totalWeight - 100) > 0.01) { + return NextResponse.json( + { + success: false, + error: `Weights must sum to 100% (current: ${totalWeight.toFixed(1)}%)`, + }, + { status: 400 } + ); + } + + // Validate capital allocation if provided + if ( + body.capitalAllocation !== undefined && + (typeof body.capitalAllocation !== 'number' || body.capitalAllocation <= 0) + ) { + return NextResponse.json( + { success: false, error: 'Capital allocation must be a positive number' }, + { status: 400 } + ); + } + + // Validate symbols if provided + if (body.symbols !== undefined && !Array.isArray(body.symbols)) { + return NextResponse.json( + { success: false, error: 'Symbols must be an array' }, + { status: 400 } + ); + } + + // Start optimization job + const jobId = await startOptimization({ + weights: { pnl, sharpe, drawdown }, + capitalAllocation: body.capitalAllocation, + symbols: body.symbols, + }); + + return NextResponse.json({ + success: true, + jobId, + estimatedDuration: '10-30 minutes', + message: 'Optimization started successfully', + }); + } catch (error) { + console.error('Error starting optimization:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Failed to start optimization', + }, + { status: 500 } + ); + } +}); + diff --git a/src/app/api/optimizer/status/route.ts b/src/app/api/optimizer/status/route.ts new file mode 100644 index 0000000..473094e --- /dev/null +++ b/src/app/api/optimizer/status/route.ts @@ -0,0 +1,78 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { withAuth } from '@/lib/auth/with-auth'; +import { getJobStatus } from '@/lib/services/optimizerService'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * GET /api/optimizer/status?jobId=opt_123abc + * + * Get the current status of an optimization job + * Designed for 5-second polling interval + * + * Response: + * { + * success: true, + * job: { + * jobId: "opt_123abc", + * status: "running" | "completed" | "failed" | "cancelled", + * progress: 65, + * currentStage: "Analyzing SOLUSDT (5/9)", + * startTime: 1234567890, + * estimatedTimeRemaining: 720000, + * results: { ... } // only when completed + * } + * } + */ +export const GET = withAuth(async (request: NextRequest) => { + try { + const searchParams = request.nextUrl.searchParams; + const jobId = searchParams.get('jobId'); + + if (!jobId) { + return NextResponse.json( + { success: false, error: 'Missing jobId parameter' }, + { status: 400 } + ); + } + + const job = getJobStatus(jobId); + + if (!job) { + return NextResponse.json( + { success: false, error: 'Job not found' }, + { status: 404 } + ); + } + + // Calculate elapsed time + const elapsedTime = Date.now() - job.startTime; + + return NextResponse.json({ + success: true, + job: { + jobId: job.jobId, + status: job.status, + progress: job.progress, + currentStage: job.currentStage, + startTime: job.startTime, + elapsedTime, + estimatedTimeRemaining: job.estimatedTimeRemaining, + error: job.error, + // Only include results if job is completed + results: job.status === 'completed' ? job.results : undefined, + }, + }); + } catch (error) { + console.error('Error fetching optimization status:', error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : 'Failed to fetch status', + }, + { status: 500 } + ); + } +}); + diff --git a/src/components/optimizer/OptimizerButton.tsx b/src/components/optimizer/OptimizerButton.tsx new file mode 100644 index 0000000..a704fae --- /dev/null +++ b/src/components/optimizer/OptimizerButton.tsx @@ -0,0 +1,201 @@ +'use client'; + +import React, { useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { Badge } from '@/components/ui/badge'; +import { toast } from 'sonner'; +import { Zap, Loader2, AlertTriangle, RotateCcw } from 'lucide-react'; +import { OptimizerDialog } from './OptimizerDialog'; +import { OptimizerProgressBar } from './OptimizerProgressBar'; + +/** + * OptimizerButton Component + * + * Main trigger button for the configuration optimizer + * Placed in the top right of the dashboard, next to Active Symbols + */ +export function OptimizerButton() { + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [jobId, setJobId] = useState(null); + const [jobStatus, setJobStatus] = useState<'running' | 'completed' | 'failed' | 'cancelled' | null>(null); + const [results, setResults] = useState(null); + const [isResetting, setIsResetting] = useState(false); + const [progress, setProgress] = useState(0); + + const handleOpenDialog = () => { + setIsDialogOpen(true); + }; + + const handleCloseDialog = () => { + setIsDialogOpen(false); + }; + + const handleOptimizationComplete = (_improvementPercent: number) => { + setJobStatus('completed'); + }; + + const handleOptimizationStart = () => { + setJobStatus('running'); + setResults(null); + setProgress(0); + // Don't clear jobId here - it will be set by handleJobIdChange when the job starts + }; + + const handleOptimizationCancel = () => { + setJobStatus(null); + setJobId(null); + setResults(null); + setProgress(0); + }; + + const handleJobIdChange = (nextJobId: string | null) => { + setJobId(nextJobId); + if (nextJobId === null) { + setJobStatus(null); + } else { + // When a new job starts, ensure status is set to running + setJobStatus('running'); + } + }; + + const handleResultsChange = (nextResults: any | null) => { + setResults(nextResults); + if (nextResults) { + setJobStatus('completed'); + } else { + setJobStatus(null); + setJobId(null); + } + }; + + const handleOptimizationError = (message: string) => { + setJobStatus('failed'); + setJobId(null); + setResults(null); + setProgress(0); + if (!isDialogOpen) { + toast.error('Optimization failed', { + description: message, + }); + } + }; + + const handleReset = async () => { + setIsResetting(true); + + // Clear local state FIRST to unmount progress bar immediately + setJobId(null); + setJobStatus(null); + setResults(null); + setProgress(0); + + try { + const response = await fetch('/api/optimizer/reset', { + method: 'POST', + }); + + if (!response.ok) { + throw new Error('Failed to reset optimizer'); + } + + toast.success('Optimizer reset', { + description: 'All cached results cleared. Ready for a fresh optimization.', + }); + } catch (error) { + toast.error('Reset failed', { + description: error instanceof Error ? error.message : 'Failed to reset optimizer state', + }); + } finally { + setIsResetting(false); + } + }; + + const hasResults = Boolean(results); + const isRunning = jobStatus === 'running'; + const hasError = jobStatus === 'failed'; + const improvement = typeof results?.summary?.improvementPercent === 'number' + ? results.summary.improvementPercent + : null; + + return ( + <> +
+ + + {(hasResults || hasError || isRunning) && ( + + )} +
+ + {jobId && isRunning && ( +
+ +
+ )} + + + + ); +} diff --git a/src/components/optimizer/OptimizerDialog.tsx b/src/components/optimizer/OptimizerDialog.tsx new file mode 100644 index 0000000..57ba91d --- /dev/null +++ b/src/components/optimizer/OptimizerDialog.tsx @@ -0,0 +1,330 @@ +'use client'; + +import React, { useState, useEffect } from 'react'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { Button } from '@/components/ui/button'; +import { toast } from 'sonner'; +import { Zap, Settings, BarChart3, CheckCircle2, Download } from 'lucide-react'; +import { OptimizerWeightSliders } from './OptimizerWeightSliders'; +import { OptimizerProgressBar } from './OptimizerProgressBar'; +import { OptimizerResults } from './OptimizerResults'; +import { OptimizerInfoTooltip } from './OptimizerInfoTooltip'; +import { useConfig } from '@/components/ConfigProvider'; + +interface OptimizerDialogProps { + isOpen: boolean; + onClose: () => void; + onOptimizationComplete: (improvementPercent: number) => void; + onOptimizationStart: () => void; + onOptimizationCancel: () => void; + jobId: string | null; + onJobIdChange: (jobId: string | null) => void; + results: any | null; + onResultsChange: (results: any | null) => void; +} + +/** + * OptimizerDialog Component + * + * Main modal with 3 tabs: + * 1. Configuration - Set weights and parameters + * 2. Progress - Real-time optimization progress + * 3. Results - Before/After comparison and apply button + */ +export function OptimizerDialog({ + isOpen, + onClose, + onOptimizationComplete, + onOptimizationStart, + onOptimizationCancel, + jobId, + onJobIdChange, + results, + onResultsChange, +}: OptimizerDialogProps) { + const { reloadConfig } = useConfig(); + const [activeTab, setActiveTab] = useState<'config' | 'progress' | 'results'>('config'); + + // Weight configuration + const [pnlWeight, setPnlWeight] = useState(50); + const [sharpeWeight, setSharpeWeight] = useState(30); + const [drawdownWeight, setDrawdownWeight] = useState(20); + + const [isStarting, setIsStarting] = useState(false); + const [isApplying, setIsApplying] = useState(false); + + // Reset to config tab when dialog opens + useEffect(() => { + if (isOpen && !jobId) { + setActiveTab('config'); + onResultsChange(null); + } + }, [isOpen, jobId, onResultsChange]); + + useEffect(() => { + if (isOpen && results) { + setActiveTab('results'); + } + }, [isOpen, results]); + + // Prevent closing during optimization + useEffect(() => { + if (jobId && activeTab === 'progress') { + const handleBeforeUnload = (e: BeforeUnloadEvent) => { + e.preventDefault(); + e.returnValue = ''; + }; + + window.addEventListener('beforeunload', handleBeforeUnload); + return () => window.removeEventListener('beforeunload', handleBeforeUnload); + } + }, [jobId, activeTab]); + + const handleStartOptimization = async () => { + setIsStarting(true); + onOptimizationStart(); + + try { + const response = await fetch('/api/optimizer/run', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + weights: { + pnl: pnlWeight, + sharpe: sharpeWeight, + drawdown: drawdownWeight, + }, + }), + }); + + const data = await response.json(); + + if (!data.success) { + throw new Error(data.error || 'Failed to start optimization'); + } + + onJobIdChange(data.jobId); + setActiveTab('progress'); + + toast.success('Optimization Started', { + description: data.message || 'Your configuration is being optimized...', + }); + } catch (error) { + console.error('Error starting optimization:', error); + toast.error('Failed to start optimization', { + description: error instanceof Error ? error.message : 'An unexpected error occurred', + }); + onOptimizationCancel(); + } finally { + setIsStarting(false); + } + }; + + const handleOptimizationComplete = (optimizationResults: any) => { + onResultsChange(optimizationResults); + setActiveTab('results'); + + const improvementPercent = optimizationResults.summary.improvementPercent || 0; + onOptimizationComplete(improvementPercent); + + toast.success('Optimization Complete!', { + description: `Found ${optimizationResults.summary.dailyImprovement >= 0 ? '+' : ''}$${optimizationResults.summary.dailyImprovement.toFixed(2)}/day improvement`, + }); + }; + + const handleOptimizationError = (error: string) => { + toast.error('Optimization Failed', { + description: error, + }); + onOptimizationCancel(); + onJobIdChange(null); + onResultsChange(null); + setActiveTab('config'); + }; + + const handleCancel = () => { + onOptimizationCancel(); + onJobIdChange(null); + onResultsChange(null); + setActiveTab('config'); + }; + + const handleApplyChanges = async () => { + if (!jobId) return; + + setIsApplying(true); + + try { + const response = await fetch('/api/optimizer/apply', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ jobId }), + }); + + const data = await response.json(); + + if (!data.success) { + throw new Error(data.error || 'Failed to apply configuration'); + } + + toast.success('Configuration Applied', { + description: `${data.message}\nBackup saved: ${data.backupPath}`, + }); + + // Reload config to reflect changes in UI + await reloadConfig(); + + // Reset and close + onJobIdChange(null); + onResultsChange(null); + onClose(); + } catch (error) { + console.error('Error applying configuration:', error); + toast.error('Failed to apply configuration', { + description: error instanceof Error ? error.message : 'An unexpected error occurred', + }); + } finally { + setIsApplying(false); + } + }; + + const handleDownloadResults = () => { + if (!results) return; + + const dataStr = JSON.stringify(results, null, 2); + const dataBlob = new Blob([dataStr], { type: 'application/json' }); + const url = URL.createObjectURL(dataBlob); + const link = document.createElement('a'); + link.href = url; + link.download = `optimization-results-${new Date().toISOString()}.json`; + link.click(); + URL.revokeObjectURL(url); + + handleCancel(); + }; + + const canStartOptimization = + Math.abs(pnlWeight + sharpeWeight + drawdownWeight - 100) < 0.1; + + return ( + + + +
+
+ + Configuration Optimizer +
+ +
+ + Optimize your trading configuration using historical backtest analysis + + + + setActiveTab(v as any)} className="flex-1 flex flex-col overflow-hidden"> + + + + Configuration + + + + Progress + + + + Results + + + +
+ +
+

Scoring Weights

+ +
+ +
+ +
+ + + + {jobId && ( +
+
+ 💡 Tip: You can safely close this dialog while the optimizer runs. Progress will continue to be visible next to the “Optimize Config” button on the main dashboard. +
+ +
+ )} + + + + {results && } + +
+ + +
+ + +
+
+ +
+ + + + ); +} + diff --git a/src/components/optimizer/OptimizerInfoTooltip.tsx b/src/components/optimizer/OptimizerInfoTooltip.tsx new file mode 100644 index 0000000..3f56da5 --- /dev/null +++ b/src/components/optimizer/OptimizerInfoTooltip.tsx @@ -0,0 +1,72 @@ +'use client'; + +import React from 'react'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { Button } from '@/components/ui/button'; +import { InfoIcon } from 'lucide-react'; + +/** + * OptimizerInfoTooltip Component + * + * Educational tooltip explaining how the optimizer works + */ +export function OptimizerInfoTooltip() { + return ( + + + + + + +
+

How the Optimizer Works

+ +
    +
  • + • + Analyzes 7 days of liquidation cascade data +
    • +
  • + • + Backtests thousands of parameter combinations per symbol +
    • +
  • + • + Tests volume thresholds, TP/SL, leverage, and trade sizes +
    • +
  • + • + Scores strategies using weighted metrics (PnL, Sharpe, Drawdown) +
    • +
  • + • + Optimizes capital allocation across symbols +
    • +
  • + • + Validates with realistic slippage & commission models +
    • +
+ +
+

+ ⏱️ Typically takes 10-30 minutes to complete +

+

+ ⚠️ Keep this page open during optimization +

+
+
+ + + + ); +} + diff --git a/src/components/optimizer/OptimizerProgressBar.tsx b/src/components/optimizer/OptimizerProgressBar.tsx new file mode 100644 index 0000000..c937f96 --- /dev/null +++ b/src/components/optimizer/OptimizerProgressBar.tsx @@ -0,0 +1,279 @@ +'use client'; + +import React, { useEffect, useState } from 'react'; +import { Progress } from '@/components/ui/progress'; +import { Button } from '@/components/ui/button'; +import { Loader2, X } from 'lucide-react'; + +interface OptimizerProgressBarProps { + jobId: string; + onComplete: (results: any) => void; + onCancel: () => void; + onError: (error: string) => void; + variant?: 'full' | 'inline'; + onProgressUpdate?: (progress: number) => void; +} + +/** + * OptimizerProgressBar Component + * + * Real-time progress tracking with 5-second polling + * Shows elapsed time, estimated remaining time, and current stage + */ +export function OptimizerProgressBar({ + jobId, + onComplete, + onCancel, + onError, + variant = 'full', + onProgressUpdate, +}: OptimizerProgressBarProps) { + const [progress, setProgress] = useState(0); + const [currentStage, setCurrentStage] = useState('Initializing...'); + const [elapsedTime, setElapsedTime] = useState(0); + const [estimatedTimeRemaining, setEstimatedTimeRemaining] = useState(null); + const [isCancelling, setIsCancelling] = useState(false); + const [isErrored, setIsErrored] = useState(false); + + useEffect(() => { + let intervalId: NodeJS.Timeout | null = null; + let stopped = false; + + const stopPolling = () => { + if (intervalId) { + clearInterval(intervalId); + intervalId = null; + } + stopped = true; + }; + + const pollStatus = async () => { + if (stopped) return; + + try { + setIsErrored(false); + + const response = await fetch(`/api/optimizer/status?jobId=${jobId}`); + + if (!response.ok) { + if (response.status === 404) { + console.warn(`[optimizer] status 404 for ${jobId}; stopping polling`); + stopPolling(); + return; + } + + throw new Error('Failed to fetch status'); + } + + const data = await response.json(); + if (!data.success) { + throw new Error(data.error || 'Unknown error'); + } + + const job = data.job; + + setProgress(job.progress); + setCurrentStage(job.currentStage); + setElapsedTime(job.elapsedTime); + setEstimatedTimeRemaining(job.estimatedTimeRemaining); + + // Notify parent of progress update + if (onProgressUpdate) { + onProgressUpdate(job.progress); + } + + if (job.status === 'completed') { + stopPolling(); + onComplete(job.results); + } else if (job.status === 'failed') { + stopPolling(); + setIsErrored(true); + onError(job.error || 'Optimization failed'); + } else if (job.status === 'cancelled') { + stopPolling(); + onCancel(); + } + } catch (error) { + if (stopped) return; + + console.error('Error polling optimization status:', error); + stopPolling(); + setIsErrored(true); + onError(error instanceof Error ? error.message : 'Failed to check status'); + } + }; + + pollStatus(); + intervalId = setInterval(pollStatus, 5000); + + return () => { + stopPolling(); + }; + }, [jobId, onComplete, onCancel, onError, onProgressUpdate]); + + const handleCancel = async () => { + setIsCancelling(true); + + try { + const response = await fetch(`/api/optimizer/cancel`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ jobId }), + }); + + if (!response.ok) { + throw new Error('Failed to cancel optimization'); + } + } catch (error) { + console.error('Error cancelling optimization:', error); + setIsCancelling(false); + setIsErrored(true); + onError(error instanceof Error ? error.message : 'Failed to cancel'); + } + }; + + const formatTime = (ms: number) => { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + + if (hours > 0) { + return `${hours}h ${minutes % 60}m`; + } + if (minutes > 0) { + return `${minutes}m ${seconds % 60}s`; + } + return `${seconds}s`; + }; + + if (variant === 'inline') { + return ( +
+
+ + {isErrored ? ( + <> + + Optimization unavailable + + ) : progress >= 100 ? ( + <> + + Optimization complete + + ) : ( + <> + + Optimization running... + + )} + + {progress.toFixed(0)}% +
+ +
+ {currentStage} + + {`Elapsed ${formatTime(elapsedTime)}`} + {estimatedTimeRemaining !== null && estimatedTimeRemaining > 0 + ? ` | ETA ${formatTime(estimatedTimeRemaining)}` + : progress >= 100 + ? ' | Ready to review' + : ''} + +
+ {!isErrored && progress < 100 && ( +
+ +
+ )} +
+ ); + } + + return ( +
+ {/* Overall Progress */} +
+
+ Overall Progress + {progress.toFixed(0)}% +
+ +
+ + {/* Current Stage */} +
+ +
+

{currentStage}

+ {isErrored && ( +

Optimization status unavailable.

+ )} +
+
+ + {/* Time Information */} +
+
+

Elapsed Time

+

{formatTime(elapsedTime)}

+
+ {estimatedTimeRemaining !== null && estimatedTimeRemaining > 0 && ( +
+

Est. Remaining

+

{formatTime(estimatedTimeRemaining)}

+
+ )} +
+ + {/* Cancel Button */} + {!isErrored && ( +
+ +
+ )} + + {/* Warning */} +
+ Keep this page open during optimization +
+
+ ); +} diff --git a/src/components/optimizer/OptimizerResults.tsx b/src/components/optimizer/OptimizerResults.tsx new file mode 100644 index 0000000..e12cfc4 --- /dev/null +++ b/src/components/optimizer/OptimizerResults.tsx @@ -0,0 +1,254 @@ +'use client'; + +import React from 'react'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { Badge } from '@/components/ui/badge'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { TrendingUp, DollarSign, Target } from 'lucide-react'; + +interface OptimizerResultsProps { + results: any; // Will be typed from optimizerService +} + +/** + * OptimizerResults Component + * + * Displays before/after comparison of optimization results + * Shows summary cards and detailed per-symbol comparison table + */ +export function OptimizerResults({ results }: OptimizerResultsProps) { + const { summary, recommendations } = results; + + const formatCurrency = (value: number) => { + return new Intl.NumberFormat('en-US', { + style: 'currency', + currency: 'USD', + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }).format(value); + }; + + const formatNumber = (value: number, decimals: number = 2) => { + return value.toFixed(decimals); + }; + + const formatPercent = (value: number) => { + return `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; + }; + + const getChangeColor = (value: number) => { + if (value > 0) return 'text-green-600 dark:text-green-400'; + if (value < 0) return 'text-red-600 dark:text-red-400'; + return 'text-muted-foreground'; + }; + + const computePercentChange = (optimized: number, current: number) => { + if (!Number.isFinite(current) || Math.abs(current) < 1e-6) { + return 0; + } + return ((optimized - current) / current) * 100; + }; + + return ( +
+ {/* Summary Cards */} +
+ + + + + Current Daily P&L + + + +

{formatCurrency(summary.currentDailyPnl)}

+ + + + + + + + Optimized Daily P&L + + + +

+ {formatCurrency(summary.optimizedDailyPnl)} +

+ + + + + + + + Daily Improvement + + + +

+ {formatCurrency(summary.dailyImprovement)} +

+ {summary.improvementPercent !== null && ( +

+ {formatPercent(summary.improvementPercent)} +

+ )} + + + + + + + + Monthly Projection + + + +

+ {formatCurrency(summary.monthlyImprovement)} +

+ + +
+ + {/* Per-Symbol Comparison Table */} + + + Per-Symbol Optimization Details + + +
+ + + + Symbol + Metric + Current + Optimized + Change + + + + {recommendations.map((rec: any) => { + const currentWindowMs = rec.settings.current.thresholdTimeWindow ?? 60000; + const optimizedWindowMs = rec.settings.optimized.thresholdTimeWindow ?? rec.settings.current.thresholdTimeWindow ?? 60000; + const currentCooldownMs = rec.settings.current.thresholdCooldown ?? 0; + const optimizedCooldownMs = rec.settings.optimized.thresholdCooldown ?? rec.settings.current.thresholdCooldown ?? 0; + + const currentWindowSec = currentWindowMs / 1000; + const optimizedWindowSec = optimizedWindowMs / 1000; + const currentCooldownSec = currentCooldownMs / 1000; + const optimizedCooldownSec = optimizedCooldownMs / 1000; + + const longThresholdDelta = rec.thresholds.optimized.long - rec.thresholds.current.long; + const shortThresholdDelta = rec.thresholds.optimized.short - rec.thresholds.current.short; + const tradeSizeDelta = rec.settings.optimized.tradeSize - rec.settings.current.tradeSize; + const leverageDelta = rec.settings.optimized.leverage - rec.settings.current.leverage; + + const longThresholdPercent = computePercentChange(rec.thresholds.optimized.long, rec.thresholds.current.long); + const shortThresholdPercent = computePercentChange(rec.thresholds.optimized.short, rec.thresholds.current.short); + const tradeSizePercent = computePercentChange(rec.settings.optimized.tradeSize, rec.settings.current.tradeSize); + const leveragePercent = computePercentChange(rec.settings.optimized.leverage, rec.settings.current.leverage); + const windowPercent = computePercentChange(optimizedWindowSec, currentWindowSec); + const cooldownPercent = computePercentChange(optimizedCooldownSec, currentCooldownSec); + + return ( + + + + {rec.symbol} + {rec.improvement.total > 0 && ( + + +{formatCurrency(rec.improvement.total)}/day + + )} + + Long Threshold + {formatNumber(rec.thresholds.current.long, 0)} + {formatNumber(rec.thresholds.optimized.long, 0)} + + {formatPercent(longThresholdPercent)} + + + + Short Threshold + {formatNumber(rec.thresholds.current.short, 0)} + {formatNumber(rec.thresholds.optimized.short, 0)} + + {formatPercent(shortThresholdPercent)} + + + + Trade Size + {formatNumber(rec.settings.current.tradeSize)} + {formatNumber(rec.settings.optimized.tradeSize)} + + {formatPercent(tradeSizePercent)} + + + + Leverage + {rec.settings.current.leverage}x + {rec.settings.optimized.leverage}x + + {formatPercent(leveragePercent)} + + + + Time Window + {formatNumber(currentWindowSec, 0)}s + {formatNumber(optimizedWindowSec, 0)}s + + {formatPercent(windowPercent)} + + + + Cooldown + {formatNumber(currentCooldownSec, 0)}s + {formatNumber(optimizedCooldownSec, 0)}s + + {formatPercent(cooldownPercent)} + + + + TP / SL + + {rec.settings.current.tpPercent}% / {rec.settings.current.slPercent}% + + + {rec.settings.optimized.tpPercent}% / {rec.settings.optimized.slPercent}% + + - + + + Daily PnL + + {formatCurrency(rec.settings.current.tradeSize * 0.01)} + + + {formatCurrency(rec.improvement.total + rec.settings.current.tradeSize * 0.01)} + + + {formatCurrency(rec.improvement.total)} + + + + ); + })} + +
+
+ + +
+ ); +} diff --git a/src/components/optimizer/OptimizerWeightSliders.tsx b/src/components/optimizer/OptimizerWeightSliders.tsx new file mode 100644 index 0000000..2e5b712 --- /dev/null +++ b/src/components/optimizer/OptimizerWeightSliders.tsx @@ -0,0 +1,114 @@ +'use client'; + +import React, { useEffect } from 'react'; +import { Label } from '@/components/ui/label'; +import { Slider } from '@/components/ui/slider'; + +interface OptimizerWeightSlidersProps { + pnlWeight: number; + sharpeWeight: number; + drawdownWeight: number; + onPnlWeightChange: (value: number) => void; + onSharpeWeightChange: (value: number) => void; + onDrawdownWeightChange: (value: number) => void; +} + +/** + * OptimizerWeightSliders Component + * + * Customizable weight controls for PnL/Sharpe/Drawdown scoring + * Auto-normalizes to ensure total equals 100% + */ +export function OptimizerWeightSliders({ + pnlWeight, + sharpeWeight, + drawdownWeight, + onPnlWeightChange, + onSharpeWeightChange, + onDrawdownWeightChange, +}: OptimizerWeightSlidersProps) { + const total = pnlWeight + sharpeWeight + drawdownWeight; + const isValid = Math.abs(total - 100) < 0.1; + + // Auto-normalize when total deviates significantly + useEffect(() => { + if (!isValid && total > 0) { + const normalizationFactor = 100 / total; + onPnlWeightChange(Math.round(pnlWeight * normalizationFactor)); + onSharpeWeightChange(Math.round(sharpeWeight * normalizationFactor)); + onDrawdownWeightChange(Math.round(drawdownWeight * normalizationFactor)); + } + }, [total, isValid, pnlWeight, sharpeWeight, drawdownWeight, onPnlWeightChange, onSharpeWeightChange, onDrawdownWeightChange]); + + return ( +
+
+
+ + {pnlWeight}% +
+ onPnlWeightChange(value[0])} + className="w-full" + /> +

+ Prioritize total profit generation +

+
+ +
+
+ + {sharpeWeight}% +
+ onSharpeWeightChange(value[0])} + className="w-full" + /> +

+ Prioritize consistency & risk-adjusted returns +

+
+ +
+
+ + {drawdownWeight}% +
+ onDrawdownWeightChange(value[0])} + className="w-full" + /> +

+ Prioritize capital preservation +

+
+ +
+
+ Total: + + {total.toFixed(1)}% + {!isValid && ' (Must equal 100%)'} + +
+
+
+ ); +} + diff --git a/src/lib/services/optimizerService.ts b/src/lib/services/optimizerService.ts new file mode 100644 index 0000000..372eb8d --- /dev/null +++ b/src/lib/services/optimizerService.ts @@ -0,0 +1,449 @@ +/** + * Optimizer Service + * + * Optimizer service to support UI-driven optimization + * with progress tracking and job management. + * + * @server-only This file should only run on the server + */ +import 'server-only'; +import { loadConfig, saveConfig } from '@/lib/bot/config'; +import fs from 'fs'; +import path from 'path'; +import { spawn } from 'child_process'; +const JOBS_STATE_PATH = path.join(process.cwd(), 'data', 'optimizer-jobs.json'); +const OPTIMIZATION_RESULTS_PATH = path.join(process.cwd(), 'optimization-results.json'); +import { errorLogger } from '@/lib/services/errorLogger'; +// Job state management +interface OptimizationJob { + jobId: string; + status: 'queued' | 'running' | 'completed' | 'failed' | 'cancelled'; + progress: number; // 0-100 + currentStage: string; + startTime: number; + estimatedTimeRemaining?: number; + results?: OptimizationResults; + error?: string; + config: OptimizationConfig; +} +interface OptimizationConfig { + weights: { + pnl: number; + sharpe: number; + drawdown: number; + }; + capitalAllocation?: number; + symbols?: string[]; +} +interface OptimizationResults { + timestamp: string; + summary: { + currentDailyPnl: number; + optimizedDailyPnl: number; + dailyImprovement: number; + monthlyImprovement: number; + improvementPercent: number | null; + recommendedMaxOpenPositions: number; + }; + recommendations: SymbolRecommendation[]; + capitalAllocation: any; + optimizedConfig: any; +} +interface SymbolRecommendation { + symbol: string; + thresholds: { + current: { long: number; short: number }; + optimized: { long: number; short: number }; + }; + settings: { + current: any; + optimized: any; + }; + improvement: { + long: number; + short: number; + total: number; + }; + performance: any; + scoring: any; +} +// In-memory job storage (use Redis for production multi-instance) +// Persist map on globalThis so route handlers share state during dev +const globalForOptimizerJobs = globalThis as typeof globalThis & { __optimizerJobs?: Map }; +const jobs = globalForOptimizerJobs.__optimizerJobs ?? (globalForOptimizerJobs.__optimizerJobs = new Map()); +hydrateJobsFromDisk(); +function hydrateJobsFromDisk(): void { + try { + if (!fs.existsSync(JOBS_STATE_PATH)) { + return; + } + const raw = fs.readFileSync(JOBS_STATE_PATH, 'utf8'); + if (!raw.trim()) { + return; + } + const parsed = JSON.parse(raw) as OptimizationJob[]; + for (const job of parsed) { + jobs.set(job.jobId, job); + } + } catch (error) { + console.error('Failed to hydrate optimizer jobs from disk', error); + } +} +function persistJobsToDisk(): void { + try { + const dir = path.dirname(JOBS_STATE_PATH); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + const serialized = JSON.stringify(Array.from(jobs.values()), null, 2); + fs.writeFileSync(JOBS_STATE_PATH, serialized); + } catch (error) { + console.error('Failed to persist optimizer jobs to disk', error); + } +} +const OPTIMIZER_TIMEOUT = 60 * 60 * 1000; // 1 hour +/** + * Generate unique job ID + */ +function generateJobId(): string { + return `opt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; +} +/** + * Update job progress + */ +function updateJobProgress( + jobId: string, + progress: number, + stage: string, + estimatedTimeRemaining?: number +) { + const job = jobs.get(jobId); + if (!job) return; + job.progress = Math.min(100, Math.max(0, progress)); + job.currentStage = stage; + if (estimatedTimeRemaining !== undefined) { + job.estimatedTimeRemaining = estimatedTimeRemaining; + } + jobs.set(jobId, job); + persistJobsToDisk(); +} +/** + * Start optimization job + */ +export async function startOptimization( + config: OptimizationConfig +): Promise { + console.log(`[optimizer] startOptimization request (cached jobs: ${jobs.size})`); + const jobId = generateJobId(); + // Validate weights + const totalWeight = config.weights.pnl + config.weights.sharpe + config.weights.drawdown; + if (Math.abs(totalWeight - 100) > 0.01) { + throw new Error(`Weights must sum to 100% (current: ${totalWeight}%)`); + } + // Create job entry + const job: OptimizationJob = { + jobId, + status: 'queued', + progress: 0, + currentStage: 'Initializing...', + startTime: Date.now(), + config, + }; + jobs.set(jobId, job); + persistJobsToDisk(); + // Start optimization in background + runOptimization(jobId).catch((error) => { + const job = jobs.get(jobId); + if (job) { + job.status = 'failed'; + job.error = error.message; + jobs.set(jobId, job); + persistJobsToDisk(); + } + }); + return jobId; +} +/** + * Get job status + */ +export function getJobStatus(jobId: string): OptimizationJob | null { + let job = jobs.get(jobId) || null; + if (!job) { + hydrateJobsFromDisk(); + job = jobs.get(jobId) || null; + } + if (!job) { + console.warn(`[optimizer] status miss for ${jobId} (jobs cached: ${jobs.size})`); + } + return job; +} +/** + * Cancel optimization job + */ +export function cancelJob(jobId: string): boolean { + const job = jobs.get(jobId); + if (!job || job.status === 'completed' || job.status === 'failed') { + return false; + } + job.status = 'cancelled'; + job.currentStage = 'Cancelled by user'; + jobs.set(jobId, job); + persistJobsToDisk(); + return true; +} +/** + * Apply optimized configuration + */ +export async function applyOptimizedConfig(jobId: string): Promise<{ + success: boolean; + backupPath?: string; + error?: string; +}> { + const job = jobs.get(jobId); + if (!job) { + return { success: false, error: 'Job not found' }; + } + if (job.status !== 'completed') { + return { success: false, error: 'Job not completed' }; + } + if (!job.results?.optimizedConfig) { + return { success: false, error: 'No optimized config available' }; + } + try { + // Load current config + const currentConfig = await loadConfig(); + // Create backup + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const backupPath = `config.user.backup-${timestamp}.json`; + fs.writeFileSync(backupPath, JSON.stringify(currentConfig, null, 2)); + // Apply optimized config + await saveConfig(job.results.optimizedConfig); + return { success: true, backupPath }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} +/** + * Main optimization logic + * Runs the optimization by executing an external optimizer script + * and monitoring its progress through output parsing + */ +async function runOptimization(jobId: string): Promise { + const job = jobs.get(jobId); + if (!job) return; + try { + // Set timeout + const timeoutId = setTimeout(() => { + const job = jobs.get(jobId); + if (job && job.status === 'running') { + job.status = 'failed'; + job.error = 'Optimization timed out after 1 hour'; + jobs.set(jobId, job); + persistJobsToDisk(); + } + }, OPTIMIZER_TIMEOUT); + job.status = 'running'; + job.progress = 0; + job.currentStage = 'Loading configuration...'; + jobs.set(jobId, job); + persistJobsToDisk(); + // Load current config to get symbol count + updateJobProgress(jobId, 5, 'Loading configuration...'); + const currentConfig = await loadConfig(); + const _symbolsToOptimize = job.config.symbols || Object.keys(currentConfig.symbols || {}); + updateJobProgress(jobId, 10, 'Starting optimization engine...'); + // Set environment variables for auto-confirm + const env = { + ...process.env, + FORCE_OPTIMIZER_OVERWRITE: '0', // Don't auto-apply in subprocess + FORCE_OPTIMIZER_CONFIRM: '0' + }; + const { pnl: weightPnl, sharpe: weightSharpe, drawdown: weightDrawdown } = job.config.weights; + env.OPTIMIZER_WEIGHT_PNL = String(weightPnl); + env.OPTIMIZER_WEIGHT_SHARPE = String(weightSharpe); + env.OPTIMIZER_WEIGHT_DRAWDOWN = String(weightDrawdown); + const optimizerScriptPath = path.join(process.cwd(), 'optimize-config.js'); + if (!fs.existsSync(optimizerScriptPath)) { + throw new Error(`Optimizer script not found at ${optimizerScriptPath}`); + } + // Spawn the optimizer by running the Node script directly. Using process.execPath + // keeps the invocation cross-platform and avoids Windows spawn() EINVAL errors + // that occur when launching npm.cmd without a shell under Node 24+. + const optimizerProcess = spawn(process.execPath, [optimizerScriptPath], { + env, + stdio: ['pipe', 'pipe', 'pipe'], + cwd: process.cwd() + }); + let currentSymbolIndex = 0; + // Parse output to track progress + optimizerProcess.stdout.on('data', (data: Buffer) => { + const output = data.toString(); + // Parse for progress indicators + // Look for "Analyzing X (Y/Z)" pattern + const symbolMatch = output.match(/Analyzing (\w+) \((\d+)\/(\d+)\)/i); + if (symbolMatch) { + const symbolName = symbolMatch[1]; + currentSymbolIndex = parseInt(symbolMatch[2]); + const total = parseInt(symbolMatch[3]); + + // Progress: 15% start + 70% for symbols + 15% finalization + const symbolProgress = 15 + (currentSymbolIndex / total) * 70; + + // Estimate remaining time + const elapsed = Date.now() - job.startTime; + const estimated = (elapsed / symbolProgress) * (100 - symbolProgress); + + updateJobProgress( + jobId, + symbolProgress, + `Analyzing ${symbolName} (${currentSymbolIndex}/${total})...`, + estimated + ); + } + // Look for completion indicators + if (output.includes('Optimization complete') || output.includes('Results saved')) { + updateJobProgress(jobId, 95, 'Finalizing results...'); + } + }); + optimizerProcess.stderr.on('data', (data: Buffer) => { + const output = data.toString(); + console.error('Optimizer stderr:', output); + errorLogger + .logError(new Error(output), { + type: 'system', + severity: 'high', + context: { + component: 'optimizer', + metadata: { + jobId, + stream: 'stderr' + } + } + }) + .catch((logError) => { + console.error('Failed to log optimizer stderr', logError); + }); + }); + // Wait for process to complete + await new Promise((resolve, reject) => { + optimizerProcess.on('close', (code: number) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Optimizer exited with code ${code}`)); + } + }); + optimizerProcess.on('error', (error: Error) => { + reject(error); + }); + // Check for cancellation + const checkInterval = setInterval(() => { + const currentJob = jobs.get(jobId); + if (currentJob?.status === 'cancelled') { + clearInterval(checkInterval); + optimizerProcess.kill(); + reject(new Error('Optimization cancelled by user')); + } + }, 1000); + optimizerProcess.on('close', () => { + clearInterval(checkInterval); + }); + }); + // Load the results from optimization-results.json + updateJobProgress(jobId, 98, 'Loading results...'); + + const resultsPath = 'optimization-results.json'; + + if (!fs.existsSync(resultsPath)) { + throw new Error('Optimization results file not found'); + } + const resultsData = fs.readFileSync(resultsPath, 'utf8'); + const results: OptimizationResults = JSON.parse(resultsData); + // Complete job + updateJobProgress(jobId, 100, 'Optimization complete!'); + job.status = 'completed'; + job.results = results; + jobs.set(jobId, job); + persistJobsToDisk(); + clearTimeout(timeoutId); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + job.status = 'failed'; + job.error = message; + jobs.set(jobId, job); + persistJobsToDisk(); + errorLogger + .logError(error instanceof Error ? error : new Error(message), { + type: 'system', + severity: 'high', + context: { + component: 'optimizer', + metadata: { + jobId, + message + } + } + }) + .catch((logError) => { + console.error('Failed to log optimizer failure', logError); + }); + } +} +/** + * Clean up old jobs (call periodically) + */ +export function cleanupOldJobs(maxAgeMs: number = 24 * 60 * 60 * 1000): number { + let cleaned = 0; + let mutated = false; + const now = Date.now(); + for (const [jobId, job] of jobs.entries()) { + if (now - job.startTime > maxAgeMs) { + jobs.delete(jobId); + cleaned++; + mutated = true; + } + } + if (mutated) { + persistJobsToDisk(); + } + return cleaned; +} +export function resetOptimizerState(): void { + for (const job of jobs.values()) { + if (job.status === 'running') { + job.status = 'cancelled'; + job.currentStage = 'Reset by user'; + job.error = 'Optimization reset by user'; + } + } + + jobs.clear(); + + try { + if (fs.existsSync(JOBS_STATE_PATH)) { + fs.unlinkSync(JOBS_STATE_PATH); + } + } catch (error) { + console.error('Failed to remove optimizer jobs cache', error); + } + + try { + if (fs.existsSync(OPTIMIZATION_RESULTS_PATH)) { + fs.unlinkSync(OPTIMIZATION_RESULTS_PATH); + } + } catch (error) { + console.error('Failed to remove optimization results file', error); + } +} + +// Export types for API routes +export type { + OptimizationJob, + OptimizationConfig, + OptimizationResults, + SymbolRecommendation, +}; + From 922c3c1f29d7bbc9ff666fc56322b7e9f0a5a587 Mon Sep 17 00:00:00 2001 From: fciaf420 Date: Wed, 8 Oct 2025 16:20:49 -0400 Subject: [PATCH 2/5] feat: expand thorough optimizer coverage Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- optimize-config.js | 774 ++++++++++++++++-- src/app/api/optimizer/reset/route.ts | 6 +- src/app/api/optimizer/run/route.ts | 11 +- src/app/api/optimizer/status/route.ts | 1 + src/app/page.tsx | 56 +- src/components/optimizer/OptimizerButton.tsx | 29 +- src/components/optimizer/OptimizerDialog.tsx | 80 +- .../optimizer/OptimizerInfoTooltip.tsx | 1 + .../optimizer/OptimizerProgressBar.tsx | 36 +- src/components/optimizer/OptimizerResults.tsx | 37 +- .../optimizer/OptimizerWeightSliders.tsx | 1 + src/lib/services/optimizerService.ts | 106 ++- 12 files changed, 978 insertions(+), 160 deletions(-) diff --git a/optimize-config.js b/optimize-config.js index 2c00c1c..3ac47eb 100644 --- a/optimize-config.js +++ b/optimize-config.js @@ -150,6 +150,33 @@ async function getAccountBalance(credentials) { } } +async function getLeverageBrackets(credentials) { + try { + const queryString = buildSignedQuery({}, credentials); + const response = await axios.get( + `https://fapi.asterdex.com/fapi/v1/leverageBracket?${queryString}`, + { + headers: { 'X-MBX-APIKEY': credentials.apiKey }, + timeout: API_TIMEOUT_MS + } + ); + + // Convert to a lookup map: { 'BTCUSDT': [...brackets], 'ETHUSDT': [...brackets] } + const bracketsMap = {}; + if (Array.isArray(response.data)) { + for (const item of response.data) { + if (item.symbol && Array.isArray(item.brackets)) { + bracketsMap[item.symbol] = item.brackets; + } + } + } + return bracketsMap; + } catch (error) { + console.error('⚠️ Failed to fetch leverage brackets:', error.response?.data || error.message); + return {}; + } +} + async function getAccountInfo(credentials) { try { const queryString = buildSignedQuery({}, credentials); @@ -167,6 +194,70 @@ async function getAccountInfo(credentials) { } } +// Leverage tier validation helpers +function getMaxLeverageForNotional(symbol, notionalValue, leverageBrackets) { + const brackets = leverageBrackets[symbol]; + if (!brackets || !Array.isArray(brackets)) { + // No tier data available - return conservative default + return 10; + } + + // Find the bracket that this notional value falls into + for (const bracket of brackets) { + const floor = bracket.notionalFloor || 0; + const cap = bracket.notionalCap || Infinity; + + if (notionalValue >= floor && notionalValue < cap) { + return bracket.initialLeverage || 10; + } + } + + // If notional exceeds all brackets, use the last bracket's leverage + if (brackets.length > 0) { + return brackets[brackets.length - 1].initialLeverage || 1; + } + + return 10; // Fallback default +} + +function isLeverageValidForNotional(symbol, leverage, notionalValue, leverageBrackets) { + const maxAllowed = getMaxLeverageForNotional(symbol, notionalValue, leverageBrackets); + return leverage <= maxAllowed; +} + +function calculateMaxPositionsForLeverage(symbol, tradeSize, leverage, maxMargin, leverageBrackets) { + const brackets = leverageBrackets[symbol]; + if (!brackets || !Array.isArray(brackets)) { + // No tier data - use simple calculation + return Math.floor(maxMargin / tradeSize); + } + + let totalPositions = 0; + let remainingMargin = maxMargin; + let cumulativeNotional = 0; + + // Simulate adding positions until we hit a tier limit or run out of margin + for (let i = 0; i < 1000; i++) { // Safety limit + const nextNotional = cumulativeNotional + (tradeSize * leverage); + + if (!isLeverageValidForNotional(symbol, leverage, nextNotional, leverageBrackets)) { + // Hit tier limit + break; + } + + if (remainingMargin < tradeSize) { + // Out of margin + break; + } + + totalPositions++; + cumulativeNotional = nextNotional; + remainingMargin -= tradeSize; + } + + return totalPositions; +} + async function getUserTrades(credentials, symbol, limit = 100, startTime = null, endTime = null) { try { const params = { symbol, limit }; @@ -417,43 +508,192 @@ const DEFAULT_THRESHOLD_WINDOW_MS = 60 * 1000; const DEFAULT_THRESHOLD_COOLDOWN_MS = 30 * 1000; const HUNTER_COOLDOWN_MS = 2 * 60 * 1000; -function generateTimeWindowCandidates(currentMs) { - const seconds = Math.max(10, Math.round((currentMs || DEFAULT_THRESHOLD_WINDOW_MS) / 1000)); - const base = [20, 30, 45, 60, 75, 90, 120, 150, 180, 240]; - const dynamic = [ - seconds, - Math.max(10, Math.round(seconds * 0.5)), - Math.max(10, Math.round(seconds * 0.75)), - Math.round(seconds * 1.25), - Math.round(seconds * 1.5), - Math.round(seconds * 2) - ]; +const optimizerModeRaw = process.env.OPTIMIZER_MODE ?? 'quick'; +const OPTIMIZER_MODE = typeof optimizerModeRaw === 'string' + ? optimizerModeRaw.trim().toLowerCase() + : 'quick'; +const MODE_SETTINGS = { + quick: { + thresholdMax: 12, + tpMax: 5, + slMax: 5, + leverageMax: 5, + marginMax: 4, + windowMax: 4, + cooldownMax: 4, + }, + thorough: { + thresholdMax: 48, + tpMax: 24, + slMax: 24, + leverageMax: 10, + marginMax: 10, + windowMax: 12, + cooldownMax: 12, + } +}; - const candidates = new Set([...base, ...dynamic]); - const filtered = [...candidates] - .filter((sec) => Number.isFinite(sec) && sec >= 10 && sec <= 300) - .sort((a, b) => a - b); +const modeSettings = MODE_SETTINGS[OPTIMIZER_MODE] || MODE_SETTINGS.quick; +const IS_THOROUGH_MODE = OPTIMIZER_MODE === 'thorough'; - return filtered.map((sec) => sec * 1000); +if (!MODE_SETTINGS[OPTIMIZER_MODE]) { + console.log(`⚙️ Optimizer mode '${OPTIMIZER_MODE}' not recognized. Falling back to 'quick'.`); } -function generateCooldownCandidates(currentMs) { - const seconds = Math.max(5, Math.round((currentMs || DEFAULT_THRESHOLD_COOLDOWN_MS) / 1000)); - const base = [5, 10, 15, 20, 30, 45, 60, 90, 120, 180]; - const dynamic = [ - seconds, - Math.max(5, Math.round(seconds * 0.5)), - Math.max(5, Math.round(seconds * 0.75)), - Math.round(seconds * 1.25), - Math.round(seconds * 1.5) - ]; +function emitProgress(percent, stage) { + if (!Number.isFinite(percent)) { + return; + } + + const clamped = Math.min(100, Math.max(0, percent)); + const label = typeof stage === 'string' && stage.trim().length > 0 + ? stage.trim() + : 'Working...'; + + console.log(`[[PROGRESS:${clamped.toFixed(2)}]] ${label}`); +} + +const MIN_THRESHOLD_PERCENTILE = (() => { + const raw = parseFloat(process.env.OPTIMIZER_MIN_THRESHOLD_PCTL || '0.35'); + if (!Number.isFinite(raw)) return 0.35; + const clamped = Math.min(Math.max(raw, 0.15), 0.6); + return parseFloat(clamped.toFixed(4)); +})(); + +const symbolGapCache = new Map(); + +function getSymbolGapStats(symbol) { + if (symbolGapCache.has(symbol)) { + return symbolGapCache.get(symbol); + } + + const rows = db.prepare(` + SELECT event_time + FROM liquidations + WHERE symbol = ? + ORDER BY event_time + `).all(symbol); + + const gaps = []; + for (let i = 1; i < rows.length; i++) { + const prev = rows[i - 1]?.event_time; + const curr = rows[i]?.event_time; + const diff = Number.isFinite(curr) && Number.isFinite(prev) + ? (curr - prev) / 1000 + : null; + + if (Number.isFinite(diff) && diff >= 0.5) { + // Ignore extremely long gaps (> 12h) which likely represent downtime + if (diff <= 12 * 60 * 60) { + gaps.push(diff); + } + } + } - const candidates = new Set([...base, ...dynamic]); - const filtered = [...candidates] - .filter((sec) => Number.isFinite(sec) && sec >= 5 && sec <= 240) - .sort((a, b) => a - b); + const avgGapSec = gaps.length + ? gaps.reduce((sum, val) => sum + val, 0) / gaps.length + : 60; - return filtered.map((sec) => sec * 1000); + const percentileValues = gaps.length + ? computePercentiles(gaps, [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98]) + : {}; + + const stats = { gaps, avgGapSec, percentiles: percentileValues }; + symbolGapCache.set(symbol, stats); + return stats; +} + +function generateTimeWindowCandidates(symbol, currentMs, volStats) { + const { gaps, avgGapSec, percentiles } = getSymbolGapStats(symbol); + const baseDefaults = [20, 30, 45, 60, 75, 90, 120, 150, 180, 240, 300]; + const candidates = new Set(baseDefaults); + + if (gaps.length) { + Object.values(percentiles).forEach((sec) => { + if (Number.isFinite(sec)) { + const clamped = Math.max(10, Math.min(Math.round(sec), 600)); + candidates.add(clamped); + } + }); + + const avgBased = [0.5, 0.75, 1, 1.25, 1.5, 2, 3].map(mult => avgGapSec * mult); + avgBased.forEach((sec) => { + if (Number.isFinite(sec)) { + const clamped = Math.max(10, Math.min(Math.round(sec), 600)); + candidates.add(clamped); + } + }); + } + + if (volStats?.perc95 && volStats.perc95 > 0) { + if (volStats.perc95 >= 3) { + [15, 20, 25, 30].forEach(sec => candidates.add(sec)); + } else if (volStats.perc95 <= 1) { + [150, 180, 210, 240, 300, 360].forEach(sec => candidates.add(sec)); + } + } + + if (Number.isFinite(currentMs) && currentMs > 0) { + candidates.add(Math.max(10, Math.round(currentMs / 1000))); + } + + const filteredSeconds = dedupeAndSort([...candidates]).filter(sec => sec >= 10 && sec <= 600); + const sampledSeconds = sampleCandidates(filteredSeconds, modeSettings.windowMax); + return sampledSeconds.map(sec => sec * 1000); +} + +function generateCooldownCandidates(symbol, currentMs, windowCandidatesMs = [], volStats) { + const { gaps, avgGapSec, percentiles } = getSymbolGapStats(symbol); + const baseDefaults = [5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300, 420, 600]; + const candidates = new Set(baseDefaults); + + if (gaps.length) { + Object.values(percentiles).forEach((sec) => { + if (Number.isFinite(sec)) { + const clamped = Math.max(5, Math.min(Math.round(sec), 900)); + candidates.add(clamped); + } + }); + + const avgBased = [1, 1.25, 1.5, 2, 3, 4].map(mult => avgGapSec * mult); + avgBased.forEach((sec) => { + if (Number.isFinite(sec)) { + const clamped = Math.max(5, Math.min(Math.round(sec), 900)); + candidates.add(clamped); + } + }); + } + + if (volStats?.perc95 && volStats.perc95 > 0) { + if (volStats.perc95 >= 3) { + [5, 10, 15, 20, 25].forEach(sec => candidates.add(sec)); + } else if (volStats.perc95 <= 1) { + [120, 180, 240, 300, 360, 420].forEach(sec => candidates.add(sec)); + } + } + + const windowSeconds = windowCandidatesMs + .map((ms) => Math.max(5, Math.round(ms / 1000))) + .filter((sec) => Number.isFinite(sec)); + + if (windowSeconds.length) { + const maxWindow = Math.max(...windowSeconds); + const minWindow = Math.min(...windowSeconds); + [0.5, 0.75, 1, 1.5, 2, 3].forEach(mult => { + const sec = Math.round(maxWindow * mult); + if (Number.isFinite(sec)) { + candidates.add(Math.max(minWindow, Math.min(sec, 900))); + } + }); + } + + if (Number.isFinite(currentMs) && currentMs > 0) { + candidates.add(Math.max(5, Math.round(currentMs / 1000))); + } + + const filteredSeconds = dedupeAndSort([...candidates]).filter(sec => sec >= 5 && sec <= 900); + const sampledSeconds = sampleCandidates(filteredSeconds, modeSettings.cooldownMax); + return sampledSeconds.map(sec => sec * 1000); } function calculateCombinationScore(longResult, shortResult) { @@ -477,6 +717,7 @@ function calculateCombinationScore(longResult, shortResult) { combinedPnl, combinedSharpe, drawdownScore, + combinedDrawdown, }; } const symbolSpanCache = new Map(); @@ -488,7 +729,11 @@ function dedupeAndSort(values) { function sampleCandidates(values, maxCount) { const sorted = dedupeAndSort(values); - if (sorted.length <= maxCount) { + if (!Number.isFinite(maxCount) || maxCount <= 0) { + return sorted; + } + + if (OPTIMIZER_MODE === 'thorough' || sorted.length <= maxCount) { return sorted; } @@ -539,29 +784,118 @@ function getLiquidationVolumes(symbol, side) { return rows.map(row => parseFloat(row.volume_usdt) || 0).filter(v => v > 0); } -function generateThresholdCandidates(symbol, side, currentThreshold) { +function generateThresholdCandidates(symbol, side, currentThreshold, maxCount = modeSettings.thresholdMax) { const volumes = getLiquidationVolumes(symbol, side); if (volumes.length === 0) { - return currentThreshold ? [currentThreshold] : []; + return { + candidates: currentThreshold ? [currentThreshold] : [], + minAllowed: currentThreshold || 0 + }; + } + + // NEW APPROACH: Generate candidates based on LIQUIDATION DATA, not user's current setting + // This ensures we explore the full realistic range regardless of user's initial config + + const candidates = []; + + // Sample across the full distribution of liquidation sizes + // Use more granular percentiles to get better coverage + const percentileTargets = [ + 0.10, 0.15, 0.20, 0.25, 0.30, + 0.35, 0.40, 0.45, 0.50, 0.55, + 0.60, 0.65, 0.70, 0.75, 0.80, + 0.85, 0.90, 0.93, 0.95, 0.97, + 0.98, 0.99 + ]; + + if (!percentileTargets.some(p => Math.abs(p - MIN_THRESHOLD_PERCENTILE) < 1e-6)) { + percentileTargets.push(MIN_THRESHOLD_PERCENTILE); } - const percentiles = computePercentiles(volumes, [0.5, 0.65, 0.75, 0.85, 0.9, 0.95, 0.98]); - const candidates = [currentThreshold]; - Object.values(percentiles).forEach(value => { - if (value && value > 0) { - // Round to nearest 10 for stability + percentileTargets.sort((a, b) => a - b); + + const percentiles = computePercentiles(volumes, percentileTargets); + + percentileTargets.forEach((p) => { + const value = percentiles[p]; + if (Number.isFinite(value) && value > 0) { candidates.push(Math.round(value / 10) * 10); } }); - if (currentThreshold) { - candidates.push(currentThreshold * 0.75); - candidates.push(currentThreshold * 0.5); - candidates.push(currentThreshold * 1.25); - candidates.push(currentThreshold * 1.5); + const tenthPercentile = percentiles[0.10]; + const minThreshold = Number.isFinite(tenthPercentile) + ? Math.round(tenthPercentile / 10) * 10 + : 0; + if (minThreshold > 0) { + candidates.push(minThreshold); + } + + // Include current threshold only for comparison, not as an anchor + if (currentThreshold && currentThreshold > 0) { + candidates.push(currentThreshold); } - return dedupeAndSort(candidates); + const floorPriority = [ + MIN_THRESHOLD_PERCENTILE, + 0.5, + 0.45, + 0.40, + 0.35, + 0.30, + 0.25 + ]; + + let minAllowedRaw = null; + for (const key of floorPriority) { + const value = percentiles[key]; + if (Number.isFinite(value) && value > 0) { + minAllowedRaw = value; + break; + } + } + + if (!Number.isFinite(minAllowedRaw) || minAllowedRaw <= 0) { + minAllowedRaw = minThreshold || currentThreshold || 0; + } + + const minAllowed = Math.max( + Math.round((minAllowedRaw || 0) / 10) * 10, + minThreshold > 0 ? minThreshold : 0 + ); + + // Add intermediate values to fill gaps between percentiles + const sorted = dedupeAndSort(candidates); + const withIntermediates = [...sorted]; + + for (let i = 0; i < sorted.length - 1; i++) { + const current = sorted[i]; + const next = sorted[i + 1]; + const gap = next - current; + + // If gap is large (> 50% of current value), add midpoint + if (gap > current * 0.5 && gap > 100) { + const midpoint = Math.round((current + next) / 2 / 10) * 10; + if (midpoint > current && midpoint < next) { + withIntermediates.push(midpoint); + } + } + } + + let sampled = sampleCandidates(withIntermediates, maxCount); + if (!IS_THOROUGH_MODE) { + sampled = sampled.filter(value => value >= minAllowed); + } + + if (!sampled.length) { + const fallback = minAllowed || currentThreshold || minThreshold || 0; + sampled = [fallback].filter(Boolean); + } + + return { + candidates: sampled, + minAllowed: minAllowed || 0 + }; } function computePriceVolatility(priceData) { @@ -603,6 +937,103 @@ function computePriceVolatility(priceData) { }; } +function computePriceRangeStats(priceData, windowSize = 60) { + if (!Array.isArray(priceData) || priceData.length === 0) { + return { + medianRange: 0, + range80: 0, + range90: 0, + range95: 0, + range98: 0 + }; + } + + const ranges = []; + const window = Math.max(1, Math.round(windowSize)); + + for (let i = 0; i < priceData.length; i++) { + const start = Math.max(0, i - window + 1); + let maxHigh = -Infinity; + let minLow = Infinity; + + for (let j = start; j <= i; j++) { + const candle = priceData[j]; + if (!candle) continue; + const high = Number.isFinite(candle.high) ? candle.high : candle.close; + const low = Number.isFinite(candle.low) ? candle.low : candle.close; + if (!Number.isFinite(high) || !Number.isFinite(low) || low <= 0) continue; + if (high > maxHigh) maxHigh = high; + if (low < minLow) minLow = low; + } + + if (!Number.isFinite(maxHigh) || !Number.isFinite(minLow) || minLow <= 0) { + continue; + } + + const rangePercent = ((maxHigh - minLow) / minLow) * 100; + if (Number.isFinite(rangePercent)) { + ranges.push(rangePercent); + } + } + + if (!ranges.length) { + return { + medianRange: 0, + range80: 0, + range90: 0, + range95: 0, + range98: 0 + }; + } + + const percentileValues = computePercentiles(ranges, [0.5, 0.8, 0.9, 0.95, 0.98]); + + return { + medianRange: percentileValues[0.5] || 0, + range80: percentileValues[0.8] || percentileValues[0.5] || 0, + range90: percentileValues[0.9] || percentileValues[0.8] || 0, + range95: percentileValues[0.95] || percentileValues[0.9] || 0, + range98: percentileValues[0.98] || percentileValues[0.95] || 0 + }; +} + +function determineMinDcaSlots(range95) { + if (!Number.isFinite(range95) || range95 <= 0) { + return 6; + } + if (range95 >= 8) return 24; + if (range95 >= 6) return 18; + if (range95 >= 4) return 12; + if (range95 >= 2) return 8; + return 6; +} + +function computeVolatilityPenalty(rangeStats, combinedDrawdown, margin, combinedPnl) { + if (!rangeStats || !Number.isFinite(rangeStats.range95)) { + return 0; + } + if (!Number.isFinite(combinedDrawdown) || !Number.isFinite(margin) || !Number.isFinite(combinedPnl)) { + return 0; + } + if (margin <= 0 || combinedDrawdown <= 0) { + return 0; + } + + const volatilityExcess = Math.max(0, rangeStats.range95 - 4); + if (volatilityExcess <= 0) { + return 0; + } + + const drawdownRatio = combinedDrawdown / margin; + const penaltyFactor = Math.max(0, drawdownRatio - 0.75); + if (penaltyFactor <= 0) { + return 0; + } + + const base = Math.max(combinedPnl, margin); + return volatilityExcess * penaltyFactor * (base * 0.1); +} + function generateTpCandidates(volStats, currentTp) { const base = Math.max(volStats.avgAbsReturn || 0.3, 0.1); const highVol = Math.max(volStats.perc95 || base * 2, base); @@ -629,7 +1060,7 @@ function generateTpCandidates(volStats, currentTp) { .map(val => Number.isFinite(val) ? parseFloat(val.toFixed(2)) : null) .filter(val => typeof val === 'number' && val > 0.05 && val <= 40); - const candidates = sampleCandidates(rawCandidates, 15) + const candidates = sampleCandidates(rawCandidates, modeSettings.tpMax) .filter(val => val >= 0.1 && val <= 30); // Basic sanity bounds only return candidates; @@ -657,25 +1088,32 @@ function generateSlCandidates(volStats, currentSl) { .map(val => Number.isFinite(val) ? parseFloat(val.toFixed(2)) : null) .filter(val => typeof val === 'number' && val > 0.1 && val <= 80); - const candidates = sampleCandidates(rawCandidates, 15) + const candidates = sampleCandidates(rawCandidates, modeSettings.slMax) .filter(val => val >= 0.1 && val <= 40); // Basic sanity bounds only return candidates; } function generateLeverageCandidates(currentLeverage) { - const baseCandidates = [currentLeverage, 5, 7.5, 10, 12.5, 15, 20, 25]; - return dedupeAndSort(baseCandidates).filter(val => val > 0 && val <= 25); + const baseCandidates = [currentLeverage, 5, 7.5, 10, 12.5, 15, 17.5, 20, 25]; + const filtered = dedupeAndSort(baseCandidates).filter(val => val > 0 && val <= 25); + return sampleCandidates(filtered, modeSettings.leverageMax); } -function generateMarginCandidates(capitalBudget, currentMargin) { +function generateMarginCandidates(capitalBudget, currentMargin, minMargin = 0) { const base = currentMargin > 0 ? currentMargin : capitalBudget * 0.5; const candidates = [currentMargin, base * 0.75, base, base * 1.25, capitalBudget * 0.5, capitalBudget * 0.75, capitalBudget]; - const sanitized = dedupeAndSort(candidates).map(val => Math.min(val, capitalBudget)); - return sanitized.filter(val => val > 0); + const clampedMin = Math.max(0, Math.min(capitalBudget, minMargin || 0)); + const sanitized = dedupeAndSort(candidates) + .map(val => { + const elevated = Math.max(val, clampedMin); + return Math.min(elevated, capitalBudget); + }) + .filter(val => val > 0); + return sampleCandidates(sanitized, modeSettings.marginMax); } -async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spanDays) { +async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spanDays, leverageBrackets = {}) { const cloneConfig = { ...symbolConfig }; const baseLongTradeSize = symbolConfig.longTradeSize ?? symbolConfig.tradeSize ?? 20; @@ -688,24 +1126,51 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa const currentTp = symbolConfig.tpPercent || 1; const currentSl = symbolConfig.slPercent || 5; const thresholdEnabled = symbolConfig.useThreshold !== false; - const currentTimeWindowMs = symbolConfig.thresholdTimeWindow || DEFAULT_THRESHOLD_WINDOW_MS; - const currentCooldownMs = symbolConfig.thresholdCooldown || DEFAULT_THRESHOLD_COOLDOWN_MS; - const timeWindowCandidates = thresholdEnabled - ? generateTimeWindowCandidates(currentTimeWindowMs) - : [currentTimeWindowMs]; - const cooldownCandidates = thresholdEnabled - ? generateCooldownCandidates(currentCooldownMs) - : [currentCooldownMs]; - const longBasePositions = Math.max(1, Math.floor(currentMargin / (baseTradeSize || 1)) || 1); const shortBasePositions = Math.max(1, Math.floor(currentMargin / (baseShortTradeSize || 1)) || 1); // Preload price data for volatility estimation const priceData = await getCachedHistoricalPrices(symbol, '1m', 10080); const volStats = computePriceVolatility(priceData); + const rangeStats = computePriceRangeStats(priceData, 60); + const minSlotsFromVolatility = determineMinDcaSlots(rangeStats.range95); + const enforceVolatilityConstraints = !IS_THOROUGH_MODE; + const requiredLongSlots = enforceVolatilityConstraints + ? Math.max(longBasePositions, minSlotsFromVolatility) + : longBasePositions; + const requiredShortSlots = enforceVolatilityConstraints + ? Math.max(shortBasePositions, minSlotsFromVolatility) + : shortBasePositions; + const baseSlotMargin = Math.max(longBasePositions * baseTradeSize, shortBasePositions * baseShortTradeSize); + const minRequiredMargin = enforceVolatilityConstraints + ? Math.max(requiredLongSlots * baseTradeSize, requiredShortSlots * baseShortTradeSize) + : baseSlotMargin; + if (enforceVolatilityConstraints && capitalBudget < minRequiredMargin) { + console.log(`⚠️ ${symbol}: Available capital (${formatNumber(capitalBudget)}) is below volatility-driven minimum margin requirement (${formatNumber(minRequiredMargin)}). Optimization will be constrained.`); + } + const applyVolatilityPenalty = !IS_THOROUGH_MODE; + + const currentTimeWindowMs = symbolConfig.thresholdTimeWindow || DEFAULT_THRESHOLD_WINDOW_MS; + const currentCooldownMs = symbolConfig.thresholdCooldown || DEFAULT_THRESHOLD_COOLDOWN_MS; + const timeWindowCandidates = thresholdEnabled + ? generateTimeWindowCandidates(symbol, currentTimeWindowMs, volStats) + : [currentTimeWindowMs]; + const cooldownCandidates = thresholdEnabled + ? generateCooldownCandidates(symbol, currentCooldownMs, timeWindowCandidates, volStats) + : [currentCooldownMs]; - const longThresholdCandidates = generateThresholdCandidates(symbol, 'SELL', currentLongThreshold || 1000); - const shortThresholdCandidates = generateThresholdCandidates(symbol, 'BUY', currentShortThreshold || 1000); + const longThresholdData = generateThresholdCandidates(symbol, 'SELL', currentLongThreshold || 1000); + const shortThresholdData = generateThresholdCandidates(symbol, 'BUY', currentShortThreshold || 1000); + const longThresholdCandidates = longThresholdData.candidates; + const shortThresholdCandidates = shortThresholdData.candidates; + const minLongThresholdAllowed = longThresholdData.minAllowed || 0; + const minShortThresholdAllowed = shortThresholdData.minAllowed || 0; + if (!longThresholdCandidates.length) { + longThresholdCandidates.push(Math.max(minLongThresholdAllowed, currentLongThreshold || 10)); + } + if (!shortThresholdCandidates.length) { + shortThresholdCandidates.push(Math.max(minShortThresholdAllowed, currentShortThreshold || 10)); + } const tpCandidatesFull = generateTpCandidates(volStats, currentTp); const slCandidatesFull = generateSlCandidates(volStats, currentSl); @@ -715,8 +1180,18 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa const slCandidates = slCandidatesFull.length > 10 ? [...slCandidatesFull.slice(0, 5), ...slCandidatesFull.slice(-5)] : slCandidatesFull; - const leverageCandidates = generateLeverageCandidates(leverageCurrent); // Test all leverage values - const marginCandidates = generateMarginCandidates(capitalBudget, currentMargin).slice(-6); // focus on higher budgets + const leverageCandidates = generateLeverageCandidates(leverageCurrent); + if (!leverageCandidates.length) { + leverageCandidates.push(leverageCurrent); + } + + const marginMinConstraint = enforceVolatilityConstraints + ? Math.min(capitalBudget, Math.max(minRequiredMargin, currentMargin)) + : Math.min(capitalBudget, Math.max(currentMargin || 0, baseSlotMargin)); + const marginCandidates = generateMarginCandidates(capitalBudget, currentMargin, marginMinConstraint); + if (!marginCandidates.length) { + marginCandidates.push(marginMinConstraint > 0 ? marginMinConstraint : Math.max(currentMargin, 1)); + } const backtestCache = new Map(); const defaultCooldownMs = Math.max(0, currentCooldownMs); @@ -774,14 +1249,22 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa const currentSharpe = (cappedCurrentLongSharpe + cappedCurrentShortSharpe) / 2; const currentDrawdown = Math.max(currentLongBacktest.maxDrawdown || 1, currentShortBacktest.maxDrawdown || 1); const currentDrawdownScore = currentTotalPnl / (currentDrawdown + 1); -// Tri-factor scoring weights are configurable via optimizer UI (defaults: 50% PnL / 30% Sharpe / 20% Drawdown) - const currentFinalScore = (currentTotalPnl * 0.5) + (currentSharpe * 0.3) + (currentDrawdownScore * 0.2); + const currentScoreRaw = ( + (currentTotalPnl * normalizedScoringWeights.pnl) + + (currentSharpe * normalizedScoringWeights.sharpe) + + (currentDrawdownScore * normalizedScoringWeights.drawdown) + ); + const currentVolatilityPenalty = computeVolatilityPenalty(rangeStats, currentDrawdown, currentMargin, currentTotalPnl); + const currentFinalScore = applyVolatilityPenalty + ? currentScoreRaw - currentVolatilityPenalty + : currentScoreRaw; let bestCombination = { totalPnl: currentTotalPnl, finalScore: currentFinalScore, sharpeRatio: currentSharpe, drawdownScore: currentDrawdownScore, + volatilityPenalty: currentVolatilityPenalty, leverage: leverageCurrent, margin: currentMargin, tp: currentTp, @@ -804,37 +1287,54 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa for (const margin of marginCandidates) { if (!Number.isFinite(margin) || margin <= 0) continue; - const longTradeSize = margin / longBasePositions; - const shortTradeSize = margin / shortBasePositions; + const longTradeSize = baseTradeSize; + const shortTradeSize = baseShortTradeSize; + if (!Number.isFinite(longTradeSize) || longTradeSize <= 0) continue; if (!Number.isFinite(shortTradeSize) || shortTradeSize <= 0) continue; + const longMaxPositions = Math.max(1, Math.floor(margin / longTradeSize)); + const shortMaxPositions = Math.max(1, Math.floor(margin / shortTradeSize)); + + if (enforceVolatilityConstraints && (longMaxPositions < requiredLongSlots || shortMaxPositions < requiredShortSlots)) { + continue; + } + + if (!Number.isFinite(longMaxPositions) || longMaxPositions <= 0) continue; + if (!Number.isFinite(shortMaxPositions) || shortMaxPositions <= 0) continue; + for (const tp of tpCandidates) { for (const sl of slCandidates) { let bestLongSide = null; for (const threshold of longThresholdCandidates) { + if (!IS_THOROUGH_MODE && threshold < minLongThresholdAllowed) { + continue; + } const candidateThreshold = Math.max(1, threshold); - const result = await runBacktest('SELL', candidateThreshold, longBasePositions, longTradeSize, leverage, tp, sl); + const result = await runBacktest('SELL', candidateThreshold, longMaxPositions, longTradeSize, leverage, tp, sl); if (!bestLongSide || result.totalPnl > bestLongSide.result.totalPnl) { bestLongSide = { threshold: candidateThreshold, result, tradeSize: longTradeSize, - maxPositions: longBasePositions + maxPositions: longMaxPositions }; } } let bestShortSide = null; for (const threshold of shortThresholdCandidates) { + if (!IS_THOROUGH_MODE && threshold < minShortThresholdAllowed) { + continue; + } const candidateThreshold = Math.max(1, threshold); - const result = await runBacktest('BUY', candidateThreshold, shortBasePositions, shortTradeSize, leverage, tp, sl); + const result = await runBacktest('BUY', candidateThreshold, shortMaxPositions, shortTradeSize, leverage, tp, sl); if (!bestShortSide || result.totalPnl > bestShortSide.result.totalPnl) { bestShortSide = { threshold: candidateThreshold, result, tradeSize: shortTradeSize, - maxPositions: shortBasePositions + maxPositions: shortMaxPositions }; } } @@ -905,12 +1405,19 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa continue; } - if (finalScore > bestCombination.finalScore) { + const volatilityPenalty = computeVolatilityPenalty(rangeStats, combinedDrawdown, margin, combinedPnl); + const adjustedScore = applyVolatilityPenalty ? (finalScore - volatilityPenalty) : finalScore; + if (!Number.isFinite(adjustedScore)) { + continue; + } + + if (adjustedScore > bestCombination.finalScore) { bestCombination = { totalPnl: combinedPnl, - finalScore: finalScore, + finalScore: adjustedScore, sharpeRatio: combinedSharpe, drawdownScore: drawdownScore, + volatilityPenalty, leverage, margin, tp, @@ -956,9 +1463,11 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa ); const metrics = calculateCombinationScore(longResult, shortResult); + const metricsPenalty = computeVolatilityPenalty(rangeStats, metrics.combinedDrawdown, bestCombination.margin, metrics.combinedPnl); + const metricsScore = applyVolatilityPenalty ? (metrics.finalScore - metricsPenalty) : metrics.finalScore; - if (metrics.finalScore > cachedBestScore + 1e-6) { - cachedBestScore = metrics.finalScore; + if (metricsScore > cachedBestScore + 1e-6) { + cachedBestScore = metricsScore; cachedBestLongResult = longResult; cachedBestShortResult = shortResult; bestWindowMs = windowMs; @@ -966,6 +1475,7 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa bestCombination.totalPnl = metrics.combinedPnl; bestCombination.sharpeRatio = metrics.combinedSharpe; bestCombination.drawdownScore = metrics.drawdownScore; + bestCombination.volatilityPenalty = metricsPenalty; } } } @@ -978,10 +1488,13 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa bestCombination.cooldownMs = bestCooldownMs; const finalMetrics = calculateCombinationScore(bestCombination.long.result, bestCombination.short.result); + const finalPenalty = computeVolatilityPenalty(rangeStats, finalMetrics.combinedDrawdown, bestCombination.margin, finalMetrics.combinedPnl); + const finalScoreWithPenalty = applyVolatilityPenalty ? (finalMetrics.finalScore - finalPenalty) : finalMetrics.finalScore; bestCombination.totalPnl = finalMetrics.combinedPnl; bestCombination.sharpeRatio = finalMetrics.combinedSharpe; bestCombination.drawdownScore = finalMetrics.drawdownScore; - bestCombination.finalScore = Math.max(bestCombination.finalScore, finalMetrics.finalScore); + bestCombination.finalScore = Math.max(bestCombination.finalScore, finalScoreWithPenalty); + bestCombination.volatilityPenalty = finalPenalty; const optimizedDailyPnl = bestCombination.totalPnl * dailyFactor; @@ -1011,8 +1524,44 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa thresholdCooldown: Math.round(bestCombination.cooldownMs || currentCooldownMs) }; + // Calculate and display max DCA positions based on leverage tiers + const desiredLongPositions = Math.max(requiredLongSlots, minSlotsFromVolatility); + const desiredShortPositions = Math.max(requiredShortSlots, minSlotsFromVolatility); + + const maxLongPositions = calculateMaxPositionsForLeverage( + symbol, + bestCombination.long.tradeSize, + bestCombination.leverage, + bestCombination.margin, + leverageBrackets + ); + const maxShortPositions = calculateMaxPositionsForLeverage( + symbol, + bestCombination.short.tradeSize, + bestCombination.leverage, + bestCombination.margin, + leverageBrackets + ); + + // Warn if tier limits would restrict DCA capacity + const tierWarning = (leverageBrackets[symbol] && (maxLongPositions < desiredLongPositions || maxShortPositions < desiredShortPositions)) + ? { + hasWarning: true, + maxLongPositions, + wantedLongPositions: desiredLongPositions, + maxShortPositions, + wantedShortPositions: desiredShortPositions, + message: `Leverage tier limits will restrict DCA positions. Max LONG: ${maxLongPositions} (wanted ${desiredLongPositions}), Max SHORT: ${maxShortPositions} (wanted ${desiredShortPositions}). Consider reducing leverage or trade size for more DCA room.` + } + : { hasWarning: false }; + + if (tierWarning.hasWarning) { + console.log(`⚠️ ${symbol}: ${tierWarning.message}\n`); + } + return { symbol, + tierWarning, current: { longThreshold: Math.max(1, currentLongThreshold), shortThreshold: Math.max(1, currentShortThreshold), @@ -1047,7 +1596,8 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa dailyPnl: optimizedDailyPnl, finalScore: bestCombination.finalScore, sharpeRatio: bestCombination.sharpeRatio, - drawdownScore: bestCombination.drawdownScore + drawdownScore: bestCombination.drawdownScore, + volatilityPenalty: bestCombination.volatilityPenalty }, improvements: { long: longImprovement, @@ -1659,16 +2209,22 @@ async function backtestSymbol(symbol, side, threshold, maxPositions, tradeSize, } // 4. Generate REALISTIC Backtest Optimization Recommendations -async function generateRecommendations(deployableCapital) { +async function generateRecommendations(deployableCapital, leverageBrackets = {}, progressBounds = {}) { console.log('REALISTIC BACKTEST OPTIMIZATION'); console.log('===================================\n'); + const { start = 55, end = 90 } = progressBounds; + const progressSpan = Math.max(0, end - start); + const recommendations = []; const optimizedConfig = JSON.parse(JSON.stringify(config)); const sanitizedCapital = Number.isFinite(deployableCapital) && deployableCapital > 0 ? deployableCapital : 0; const symbolEntries = Object.entries(config.symbols); if (symbolEntries.length === 0) { + if (progressSpan > 0) { + emitProgress(end, 'No symbols configured for optimization'); + } return { recommendations, optimizedConfig, recommendedGlobalMax: 0 }; } @@ -1682,13 +2238,22 @@ async function generateRecommendations(deployableCapital) { ? Math.max(0.25, Math.min(2.5, sanitizedCapital / baselineTotalMargin)) : 1; + const totalSymbols = symbolEntries.length; + let processedSymbols = 0; + for (const [symbol, symbolConfig] of symbolEntries) { + processedSymbols += 1; + if (progressSpan > 0) { + const symbolProgress = start + (processedSymbols / totalSymbols) * progressSpan; + emitProgress(symbolProgress, `Optimizing ${symbol} (${processedSymbols}/${totalSymbols})`); + } + console.log(`Analyzing ${symbol} (${processedSymbols}/${totalSymbols})`); const spanDays = getSymbolDataSpanDays(symbol); const fallbackMargin = (symbolConfig.tradeSize || 20) * 5; const baseMargin = symbolConfig.maxPositionMarginUSDT || fallbackMargin; const capitalBudget = Math.max(5, Math.min(sanitizedCapital || baseMargin, baseMargin * scaleFactor)); - const optimization = await optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spanDays); + const optimization = await optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spanDays, leverageBrackets); const currentDaily = optimization.current.performance.dailyPnl; const optimizedDaily = optimization.optimized.dailyPnl; @@ -1699,6 +2264,7 @@ async function generateRecommendations(deployableCapital) { recommendations.push({ symbol, + tierWarning: optimization.tierWarning, currentLong: optimization.current.longThreshold, currentShort: optimization.current.shortThreshold, optimizedLong: optimization.optimized.config.longVolumeThresholdUSDT, @@ -1750,6 +2316,10 @@ async function generateRecommendations(deployableCapital) { }; } + if (progressSpan > 0) { + emitProgress(end, 'Per-symbol optimization complete'); + } + console.log('KEY BACKTEST INSIGHTS:'); console.log('- Optimization considers thresholds, TP/SL, trade size, leverage, and margin per symbol'); console.log('- Deployable capital scaled to $' + formatLargeNumber(sanitizedCapital)); @@ -2334,39 +2904,73 @@ async function maybeApplyOptimizedConfig(originalConfig, optimizedConfig, summar async function main() { try { + emitProgress(0, 'Initializing optimizer'); + const modeLabel = MODE_SETTINGS[OPTIMIZER_MODE] ? OPTIMIZER_MODE : 'quick'; + console.log(`⚙️ Optimizer run mode: ${modeLabel === 'quick' ? 'Quick (fast sweep)' : 'Thorough (expanded search)'}\n`); + const weightSummary = `${formatWeightPercent(scoringWeights.percent.pnl)} / ${formatWeightPercent(scoringWeights.percent.sharpe)} / ${formatWeightPercent(scoringWeights.percent.drawdown)}`; const weightLabel = scoringWeights.isDefault ? ' (default)' : ''; console.log(`???? Using scoring weights (PnL / Sharpe / Drawdown): ${weightSummary}${weightLabel}\n`); console.log('???? Fetching complete account snapshot...\n'); - const [balance, accountInfo, positions] = await Promise.all([ + emitProgress(5, 'Fetching account snapshot'); + const [balance, accountInfo, positions, leverageBrackets] = await Promise.all([ getAccountBalance(config.api), getAccountInfo(config.api), - getCurrentPositions(config.api) + getCurrentPositions(config.api), + getLeverageBrackets(config.api) ]); + emitProgress(10, 'Account snapshot loaded'); + + // Display leverage tier info for key symbols + if (Object.keys(leverageBrackets).length > 0) { + console.log('📈 Leverage tiers loaded for', Object.keys(leverageBrackets).length, 'symbols\n'); + } else { + console.log('⚠️ Warning: Could not fetch leverage tiers - using conservative defaults\n'); + } // Core analyses const _avgGap = analyzePriceDataCoverage(); + emitProgress(18, 'Price data coverage analyzed'); + const capitalInfo = analyzeCapitalAllocation(balance, accountInfo, positions); + emitProgress(24, 'Account snapshot analyzed'); + analyzeLiquidationCascades(); + emitProgress(28, 'Cascade patterns analyzed'); + analyzeCurrentConfig(); + emitProgress(32, 'Current configuration performance analyzed'); + await analyzeRealTradingHistory(config.api); + emitProgress(36, 'Real trading history analyzed'); + optimizeThresholds(); + emitProgress(40, 'Threshold sweeps completed'); + rankSymbolProfitability(); + emitProgress(45, 'Symbol profitability ranked'); // Generate recommendations with backtest // Use CALCULATED TOTAL for optimal capital allocation + emitProgress(50, 'Starting per-symbol optimization'); const deployableCapital = capitalInfo.calculatedTotal || parseFloat(accountInfo?.totalWalletBalance ?? 0); - const { recommendations, optimizedConfig, recommendedGlobalMax } = await generateRecommendations(deployableCapital); + const { recommendations, optimizedConfig, recommendedGlobalMax } = await generateRecommendations(deployableCapital, leverageBrackets, { start: 55, end: 88 }); + emitProgress(90, 'Per-symbol optimization complete'); // Optimize capital allocation const capitalOptimization = optimizeCapitalAllocation(accountInfo, recommendations, optimizedConfig.symbols); + emitProgress(94, 'Capital allocation optimized'); // Generate final summary const optimizationResults = generateOptimizationSummary(recommendations, capitalOptimization, optimizedConfig, recommendedGlobalMax); + emitProgress(97, 'Optimization summary prepared'); + emitProgress(98, 'Preparing configuration updates'); await maybeApplyOptimizedConfig(config, optimizedConfig, optimizationResults.summary); + emitProgress(99, 'Finalizing optimizer run'); + emitProgress(100, 'Optimization complete'); console.log('???? Optimization analysis complete!'); const totalValue = parseFloat(accountInfo?.totalMarginBalance || balance.totalWalletBalance || 0); console.log(`???? Total account value: $${formatLargeNumber(totalValue)}`); diff --git a/src/app/api/optimizer/reset/route.ts b/src/app/api/optimizer/reset/route.ts index e440f8a..9e98ad0 100644 --- a/src/app/api/optimizer/reset/route.ts +++ b/src/app/api/optimizer/reset/route.ts @@ -19,11 +19,15 @@ export const dynamic = 'force-dynamic'; */ export const POST = withAuth(async () => { try { - resetOptimizerState(); + const outcome = resetOptimizerState(); return NextResponse.json({ success: true, message: 'Optimizer state reset successfully', + cancelledJobCount: outcome.cancelledJobCount, + clearedJobCount: outcome.clearedJobCount, + jobsFileRemoved: outcome.jobsFileRemoved, + resultsFileRemoved: outcome.resultsFileRemoved, }); } catch (error) { console.error('Error resetting optimizer state:', error); diff --git a/src/app/api/optimizer/run/route.ts b/src/app/api/optimizer/run/route.ts index a338999..c98d4aa 100644 --- a/src/app/api/optimizer/run/route.ts +++ b/src/app/api/optimizer/run/route.ts @@ -15,6 +15,7 @@ export const dynamic = 'force-dynamic'; * weights: { pnl: 50, sharpe: 30, drawdown: 20 }, * capitalAllocation?: number, * symbols?: string[] + * mode?: "quick" | "thorough" * } * * Response: @@ -80,18 +81,24 @@ export const POST = withAuth(async (request: NextRequest) => { ); } + const mode: 'quick' | 'thorough' = body.mode === 'thorough' ? 'thorough' : 'quick'; + // Start optimization job const jobId = await startOptimization({ weights: { pnl, sharpe, drawdown }, capitalAllocation: body.capitalAllocation, symbols: body.symbols, + mode, }); + const estimatedDuration = mode === 'thorough' ? '30-60 minutes' : '10-20 minutes'; + return NextResponse.json({ success: true, jobId, - estimatedDuration: '10-30 minutes', - message: 'Optimization started successfully', + estimatedDuration, + mode, + message: `Optimization started in ${mode === 'thorough' ? 'Thorough' : 'Quick'} mode`, }); } catch (error) { console.error('Error starting optimization:', error); diff --git a/src/app/api/optimizer/status/route.ts b/src/app/api/optimizer/status/route.ts index 473094e..6382ab3 100644 --- a/src/app/api/optimizer/status/route.ts +++ b/src/app/api/optimizer/status/route.ts @@ -59,6 +59,7 @@ export const GET = withAuth(async (request: NextRequest) => { startTime: job.startTime, elapsedTime, estimatedTimeRemaining: job.estimatedTimeRemaining, + mode: job.config.mode ?? 'quick', error: job.error, // Only include results if job is completed results: job.status === 'completed' ? job.results : undefined, diff --git a/src/app/page.tsx b/src/app/page.tsx index f168086..d13f28c 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -27,6 +27,7 @@ import { useWebSocketUrl } from '@/hooks/useWebSocketUrl'; import { RateLimitToastListener } from '@/hooks/useRateLimitToasts'; import dataStore, { AccountInfo, Position } from '@/lib/services/dataStore'; import { signOut } from 'next-auth/react'; +import { OptimizerButton } from '@/components/optimizer/OptimizerButton'; interface BalanceStatus { source?: string; @@ -351,33 +352,38 @@ export default function DashboardPage() {
- {/* Active Trading Symbols */} -
- -
- Active Symbols -
- {config?.symbols && Object.keys(config.symbols).length > 0 ? ( - <> - {Object.keys(config.symbols).length} -
- {Object.keys(config.symbols).slice(0, 3).map((symbol, _index) => ( - - {symbol.replace('USDT', '')} - - ))} - {Object.keys(config.symbols).length > 3 && ( - - +{Object.keys(config.symbols).length - 3} - - )} -
- - ) : ( - 0 - )} + {/* Active Trading Symbols + Optimizer Button */} +
+
+ +
+ Active Symbols +
+ {config?.symbols && Object.keys(config.symbols).length > 0 ? ( + <> + {Object.keys(config.symbols).length} +
+ {Object.keys(config.symbols).slice(0, 3).map((symbol, _index) => ( + + {symbol.replace('USDT', '')} + + ))} + {Object.keys(config.symbols).length > 3 && ( + + +{Object.keys(config.symbols).length - 3} + + )} +
+ + ) : ( + 0 + )} +
+ + {/* Optimizer Button */} +
diff --git a/src/components/optimizer/OptimizerButton.tsx b/src/components/optimizer/OptimizerButton.tsx index a704fae..cbe1fcf 100644 --- a/src/components/optimizer/OptimizerButton.tsx +++ b/src/components/optimizer/OptimizerButton.tsx @@ -94,12 +94,34 @@ export function OptimizerButton() { method: 'POST', }); - if (!response.ok) { - throw new Error('Failed to reset optimizer'); + const data = await response.json(); + + if (!response.ok || !data?.success) { + throw new Error(data?.error || 'Failed to reset optimizer'); + } + + const details: string[] = []; + + if (typeof data.cancelledJobCount === 'number' && data.cancelledJobCount > 0) { + details.push(`Cancelled ${data.cancelledJobCount} in-flight job${data.cancelledJobCount === 1 ? '' : 's'}`); + } + + if (typeof data.clearedJobCount === 'number' && data.clearedJobCount > 0) { + details.push(`Cleared ${data.clearedJobCount} cached job${data.clearedJobCount === 1 ? '' : 's'}`); + } + + if (data.jobsFileRemoved) { + details.push('Removed job cache file'); + } + + if (data.resultsFileRemoved) { + details.push('Removed previous results'); } toast.success('Optimizer reset', { - description: 'All cached results cleared. Ready for a fresh optimization.', + description: details.length + ? details.join(' • ') + : 'All cached results cleared. Ready for a fresh optimization.', }); } catch (error) { toast.error('Reset failed', { @@ -199,3 +221,4 @@ export function OptimizerButton() { ); } + diff --git a/src/components/optimizer/OptimizerDialog.tsx b/src/components/optimizer/OptimizerDialog.tsx index 57ba91d..ada6b88 100644 --- a/src/components/optimizer/OptimizerDialog.tsx +++ b/src/components/optimizer/OptimizerDialog.tsx @@ -11,11 +11,12 @@ import { import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { Button } from '@/components/ui/button'; import { toast } from 'sonner'; -import { Zap, Settings, BarChart3, CheckCircle2, Download } from 'lucide-react'; +import { Zap, Settings, BarChart3, CheckCircle2, Download, Info } from 'lucide-react'; import { OptimizerWeightSliders } from './OptimizerWeightSliders'; import { OptimizerProgressBar } from './OptimizerProgressBar'; import { OptimizerResults } from './OptimizerResults'; import { OptimizerInfoTooltip } from './OptimizerInfoTooltip'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; import { useConfig } from '@/components/ConfigProvider'; interface OptimizerDialogProps { @@ -56,6 +57,7 @@ export function OptimizerDialog({ const [pnlWeight, setPnlWeight] = useState(50); const [sharpeWeight, setSharpeWeight] = useState(30); const [drawdownWeight, setDrawdownWeight] = useState(20); + const [mode, setMode] = useState<'quick' | 'thorough'>('quick'); const [isStarting, setIsStarting] = useState(false); const [isApplying, setIsApplying] = useState(false); @@ -101,6 +103,7 @@ export function OptimizerDialog({ sharpe: sharpeWeight, drawdown: drawdownWeight, }, + mode, }), }); @@ -111,6 +114,9 @@ export function OptimizerDialog({ } onJobIdChange(data.jobId); + if (typeof data.mode === 'string') { + setMode(data.mode === 'thorough' ? 'thorough' : 'quick'); + } setActiveTab('progress'); toast.success('Optimization Started', { @@ -215,7 +221,7 @@ export function OptimizerDialog({ return ( - +
@@ -259,6 +265,69 @@ export function OptimizerDialog({ />
+
+
+
+

Run Mode

+ + + + + +
+

Quick

+

Trims the search grid for a fast 10–20 minute sweep. Ideal for daily tuning.

+

Thorough

+

Runs the full search space with deeper combos (30–60 minutes) for exhaustive analysis.

+
+ + +
+ + {mode === 'thorough' ? 'Best accuracy, longer runtime' : 'Balanced accuracy with fast runtime'} + +
+ +
+ + + + + Use a trimmed candidate grid to finish in roughly 10–20 minutes. + + + + + + Explore the full grid for maximum accuracy (30–60 minute runs). + +
+
+
@@ -177,16 +193,16 @@ export function OptimizerProgressBar({ {`Elapsed ${formatTime(elapsedTime)}`} {estimatedTimeRemaining !== null && estimatedTimeRemaining > 0 ? ` | ETA ${formatTime(estimatedTimeRemaining)}` - : progress >= 100 + : progress >= 99.9 ? ' | Ready to review' : ''}
- {!isErrored && progress < 100 && ( + {!isErrored && progress < 99.9 && (
@@ -246,7 +262,7 @@ export function OptimizerProgressBar({
{/* Cancel Button */} - {!isErrored && ( + {!isErrored && progress < 99.9 && (
+ )} + + {diagnostics && ( +
+
+

Candidate Funnel

+ {diagnostics.candidateCounts && Object.keys(diagnostics.candidateCounts).length > 0 ? ( +
+ {Object.entries(diagnostics.candidateCounts) + .sort((a, b) => (b[1] ?? 0) - (a[1] ?? 0)) + .map(([key, value]) => ( +
+ {formatLabel(key)} + {formatInteger(value)} +
+ ))} +
+ ) : ( +

No candidate counts recorded.

+ )} +
+
+ Combos Evaluated + {formatInteger(diagnostics.combinationsEvaluated)} +
+
+ Combos Accepted + {formatInteger(diagnostics.combinationsAccepted)} +
+ {Number.isFinite(diagnostics.tierAdjustments ?? NaN) && diagnostics.tierAdjustments ? ( +
+ Tier Adjustments + {formatInteger(diagnostics.tierAdjustments)} +
+ ) : null} +
+ Scenarios Evaluated + {formatInteger(diagnostics.scenariosEvaluated)} +
+
+ Runtime + {formatDuration(diagnostics.durationMs)} +
+
+
+
+

Rejection Breakdown

+ {diagnostics.rejections && Object.values(diagnostics.rejections).some((value) => value > 0) ? ( +
    + {Object.entries(diagnostics.rejections) + .filter(([, value]) => (value ?? 0) > 0) + .sort((a, b) => (b[1] ?? 0) - (a[1] ?? 0)) + .slice(0, 8) + .map(([key, value]) => ( +
  • + {formatLabel(key)} + {formatInteger(value)} +
    • + ))} +
+ ) : ( +

No rejection categories triggered.

+ )} +
+
+ Backtests Executed + {formatInteger(diagnostics.backtests?.executed)} +
+
+ Cache Hits + {formatInteger(diagnostics.backtests?.cacheHits)} +
+ {(() => { + const executed = diagnostics.backtests?.executed ?? 0; + const cacheHits = diagnostics.backtests?.cacheHits ?? 0; + const total = executed + cacheHits; + const hitRate = total > 0 ? (cacheHits / total) * 100 : null; + return ( +
+ Cache Hit Rate + {formatPercent(hitRate)} +
+ ); + })()} +
+
+
+ )} + {!risk && !diagnostics && ( +

+ No additional diagnostics were captured for this symbol. Enable the diagnostics toggle before running a thorough sweep to record candidate funnels and rejection breakdowns. +

+ )} +
+ + + ); })} diff --git a/src/lib/services/optimizerService.ts b/src/lib/services/optimizerService.ts index d1d26fd..78afa18 100644 --- a/src/lib/services/optimizerService.ts +++ b/src/lib/services/optimizerService.ts @@ -14,6 +14,7 @@ import { spawn } from 'child_process'; const JOBS_STATE_PATH = path.join(process.cwd(), 'data', 'optimizer-jobs.json'); const OPTIMIZATION_RESULTS_PATH = path.join(process.cwd(), 'optimization-results.json'); import { errorLogger } from '@/lib/services/errorLogger'; +import type { OptimizationResults, SymbolRecommendation } from '@/types/optimizer'; // Job state management interface OptimizationJob { jobId: string; @@ -35,46 +36,7 @@ interface OptimizationConfig { capitalAllocation?: number; symbols?: string[]; mode?: 'quick' | 'thorough'; -} -interface OptimizationResults { - timestamp: string; - summary: { - currentDailyPnl: number; - optimizedDailyPnl: number; - dailyImprovement: number; - monthlyImprovement: number; - improvementPercent: number | null; - recommendedMaxOpenPositions: number; - }; - recommendations: SymbolRecommendation[]; - capitalAllocation: any; - optimizedConfig: any; -} -interface SymbolRecommendation { - symbol: string; - tierWarning?: { - hasWarning: boolean; - maxLongPositions?: number; - wantedLongPositions?: number; - maxShortPositions?: number; - wantedShortPositions?: number; - message?: string; - }; - thresholds: { - current: { long: number; short: number }; - optimized: { long: number; short: number }; - }; - settings: { - current: any; - optimized: any; - }; - improvement: { - long: number; - short: number; - total: number; - }; - performance: any; - scoring: any; + diagnostics?: boolean; } // In-memory job storage (use Redis for production multi-instance) // Persist map on globalThis so route handlers share state during dev @@ -92,6 +54,7 @@ function hydrateJobsFromDisk(): void { } const parsed = JSON.parse(raw) as OptimizationJob[]; for (const job of parsed) { + job.config.symbols = sanitizeSymbolsList(job.config.symbols); jobs.set(job.jobId, job); } } catch (error) { @@ -111,6 +74,22 @@ function persistJobsToDisk(): void { } } const OPTIMIZER_TIMEOUT = 60 * 60 * 1000; // 1 hour + +function sanitizeSymbolsList(symbols?: string[]): string[] | undefined { + if (!Array.isArray(symbols)) { + return undefined; + } + + const sanitized = symbols + .map(symbol => (typeof symbol === 'string' ? symbol.trim().toUpperCase() : '')) + .filter(Boolean); + + if (sanitized.length === 0) { + return undefined; + } + + return Array.from(new Set(sanitized)); +} /** * Generate unique job ID */ @@ -145,6 +124,8 @@ export async function startOptimization( console.log(`[optimizer] startOptimization request (cached jobs: ${jobs.size})`); const jobId = generateJobId(); const mode: 'quick' | 'thorough' = config.mode === 'thorough' ? 'thorough' : 'quick'; + const diagnostics = config.diagnostics === true; + const sanitizedSymbols = sanitizeSymbolsList(config.symbols); // Validate weights const totalWeight = config.weights.pnl + config.weights.sharpe + config.weights.drawdown; if (Math.abs(totalWeight - 100) > 0.01) { @@ -160,6 +141,8 @@ export async function startOptimization( config: { ...config, mode, + diagnostics, + symbols: sanitizedSymbols, }, }; jobs.set(jobId, job); @@ -266,7 +249,10 @@ async function runOptimization(jobId: string): Promise { // Load current config to get symbol count updateJobProgress(jobId, 5, 'Loading configuration...'); const currentConfig = await loadConfig(); - const _symbolsToOptimize = job.config.symbols || Object.keys(currentConfig.symbols || {}); + const allConfigSymbols = Object.keys(currentConfig.symbols || {}); + const symbolsToOptimize = job.config.symbols && job.config.symbols.length > 0 + ? job.config.symbols + : allConfigSymbols; updateJobProgress(jobId, 10, 'Starting optimization engine...'); // Set environment variables for auto-confirm const env = { @@ -279,6 +265,12 @@ async function runOptimization(jobId: string): Promise { env.OPTIMIZER_WEIGHT_SHARPE = String(weightSharpe); env.OPTIMIZER_WEIGHT_DRAWDOWN = String(weightDrawdown); env.OPTIMIZER_MODE = job.config.mode ?? 'quick'; + env.OPTIMIZER_DIAGNOSTICS = job.config.diagnostics ? '1' : '0'; + if (symbolsToOptimize.length > 0 && symbolsToOptimize.length < allConfigSymbols.length) { + env.OPTIMIZER_SYMBOLS = symbolsToOptimize.join(','); + } else { + delete env.OPTIMIZER_SYMBOLS; + } const optimizerScriptPath = path.join(process.cwd(), 'optimize-config.js'); if (!fs.existsSync(optimizerScriptPath)) { throw new Error(`Optimizer script not found at ${optimizerScriptPath}`); diff --git a/src/types/optimizer.ts b/src/types/optimizer.ts new file mode 100644 index 0000000..1b79949 --- /dev/null +++ b/src/types/optimizer.ts @@ -0,0 +1,129 @@ +export interface ScenarioOutcome { + name: string; + totalPnl: number; + maxDrawdown: number; + sharpeRatio: number; +} + +export interface RiskProfile { + current: { + volatilityPenalty?: number | null; + cvarPenalty?: number | null; + cvar?: number | null; + payoffRatio?: number | null; + }; + optimized: { + volatilityPenalty?: number | null; + cvarPenalty?: number | null; + scenarioPenalty?: number | null; + cvar?: number | null; + payoffRatio?: number | null; + }; + scenarios?: ScenarioOutcome[]; +} + +export interface DiagnosticsSummary { + candidateCounts?: Record; + rejections?: Record; + backtests?: { + executed?: number; + cacheHits?: number; + }; + combinationsEvaluated?: number; + combinationsAccepted?: number; + scenariosEvaluated?: number; + durationMs?: number | null; + tierAdjustments?: number; +} + +export interface SymbolSettingsSnapshot { + tradeSize: number; + longTradeSize: number; + shortTradeSize: number; + maxPositionMarginUSDT?: number; + margin?: number; + leverage: number; + tpPercent: number; + slPercent: number; + maxPositionsLong?: number; + maxPositionsShort?: number; + thresholdTimeWindow?: number; + thresholdCooldown?: number; + vwapProtection?: boolean; +} + +export interface PerformanceLeg { + trades: number; + winRate: number; + pnl: number; + sharpe: number; + maxDrawdown: number; +} + +export interface PerformanceSnapshot { + long: PerformanceLeg; + short: PerformanceLeg; +} + +export interface ScoringSummary { + finalScore?: number; + sharpeRatio?: number; + drawdownScore?: number; + weights?: { + pnl: number; + sharpe: number; + drawdown: number; + }; + weightPercent?: { + pnl: number; + sharpe: number; + drawdown: number; + }; +} + +export interface SymbolRecommendation { + symbol: string; + tierWarning?: { + hasWarning: boolean; + maxLongPositions?: number; + wantedLongPositions?: number; + maxShortPositions?: number; + wantedShortPositions?: number; + message?: string; + }; + thresholds: { + current: { long: number; short: number }; + optimized: { long: number; short: number }; + }; + settings: { + current: SymbolSettingsSnapshot; + optimized: SymbolSettingsSnapshot; + }; + risk?: RiskProfile; + diagnostics?: DiagnosticsSummary | null; + improvement: { + long: number; + short: number; + total: number; + }; + performance: { + current: PerformanceSnapshot; + optimized: PerformanceSnapshot; + }; + scoring: ScoringSummary; +} + +export interface OptimizationResults { + timestamp: string; + summary: { + currentDailyPnl: number; + optimizedDailyPnl: number; + dailyImprovement: number; + monthlyImprovement: number; + improvementPercent: number | null; + recommendedMaxOpenPositions: number; + }; + recommendations: SymbolRecommendation[]; + capitalAllocation: any; + optimizedConfig: any; +} From 0e7c8b133ca607e9493087e9c4909d9419ec8781 Mon Sep 17 00:00:00 2001 From: fciaf420 Date: Thu, 9 Oct 2025 14:12:29 -0400 Subject: [PATCH 4/5] fix: restore optimizer scenario scoring Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- optimize-config.js | 4 ---- src/lib/bot/hunter.ts | 8 +++++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/optimize-config.js b/optimize-config.js index c4a21f4..32c4873 100644 --- a/optimize-config.js +++ b/optimize-config.js @@ -1734,7 +1734,6 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa }; if (diagnostics) diagnostics.scenariosEvaluated += 1; const longScenario = await runBacktest( - symbol, 'SELL', bestLongSide.threshold, bestLongSide.maxPositions, @@ -1745,7 +1744,6 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa overrides ); const shortScenario = await runBacktest( - symbol, 'BUY', bestShortSide.threshold, bestShortSide.maxPositions, @@ -1869,7 +1867,6 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa ...profile.overrides }; const longScenario = await runBacktest( - symbol, 'SELL', bestCombination.long.threshold, bestCombination.long.maxPositions, @@ -1880,7 +1877,6 @@ async function optimizeSymbolParameters(symbol, symbolConfig, capitalBudget, spa overrides ); const shortScenario = await runBacktest( - symbol, 'BUY', bestCombination.short.threshold, bestCombination.short.maxPositions, diff --git a/src/lib/bot/hunter.ts b/src/lib/bot/hunter.ts index 3c7e890..71f4e70 100644 --- a/src/lib/bot/hunter.ts +++ b/src/lib/bot/hunter.ts @@ -412,9 +412,6 @@ export class Hunter extends EventEmitter { thresholdStatus }); - const symbolConfig = this.config.symbols[liquidation.symbol]; - if (!symbolConfig) return; // Symbol not in config - const volumeUSDT = liquidation.qty * liquidation.price; // Store liquidation in database (non-blocking) @@ -434,6 +431,11 @@ export class Hunter extends EventEmitter { // Non-critical error, don't broadcast to UI to avoid spam }); + const symbolConfig = this.config.symbols[liquidation.symbol]; + if (!symbolConfig) { + return; // We're not trading this symbol, but data has been archived + } + // Check if we should use threshold system or instant trigger if (useThresholdSystem && thresholdStatus) { // NEW THRESHOLD SYSTEM - Cumulative volume in 60-second window From c194791918cada8d92e7278124e2f87b4b9b68de Mon Sep 17 00:00:00 2001 From: fciaf420 Date: Fri, 10 Oct 2025 08:48:49 -0400 Subject: [PATCH 5/5] feat: add realized pnl summary Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/app/api/pnl/realized-summary/route.ts | 154 +++++++++++++ src/app/page.tsx | 8 +- src/components/TokenRealizedPnLSummary.tsx | 256 +++++++++++++++++++++ src/lib/api/orders.ts | 9 +- src/lib/bot/hunter.ts | 51 ++-- src/lib/utils/leverage.ts | 13 ++ src/lib/validation/tradeSizeValidator.ts | 8 +- 7 files changed, 475 insertions(+), 24 deletions(-) create mode 100644 src/app/api/pnl/realized-summary/route.ts create mode 100644 src/components/TokenRealizedPnLSummary.tsx create mode 100644 src/lib/utils/leverage.ts diff --git a/src/app/api/pnl/realized-summary/route.ts b/src/app/api/pnl/realized-summary/route.ts new file mode 100644 index 0000000..1ba8170 --- /dev/null +++ b/src/app/api/pnl/realized-summary/route.ts @@ -0,0 +1,154 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { loadConfig } from '@/lib/bot/config'; +import { getIncomeHistory, IncomeRecord } from '@/lib/api/income'; + +type SummaryRange = '24h' | '7d' | '30d' | '90d' | '1y' | 'all'; + +interface SymbolSummary { + symbol: string; + realizedPnl: number; + tradeCount: number; + wins: number; + losses: number; + winRate: number; + avgWin: number; + avgLoss: number; + largestWin: number; + largestLoss: number; + lastTradeTime: number | null; +} + +const RANGE_LOOKBACK: Record = { + '24h': 24 * 60 * 60 * 1000, + '7d': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + '90d': 90 * 24 * 60 * 60 * 1000, + '1y': 365 * 24 * 60 * 60 * 1000, + all: null, +}; + +export async function GET(request: NextRequest) { + try { + const config = await loadConfig(); + const credentials = config.api; + + if (!credentials?.apiKey || !credentials?.secretKey) { + return NextResponse.json({ + symbols: [], + range: '30d', + generatedAt: Date.now(), + }); + } + + const searchParams = request.nextUrl.searchParams; + const rangeParam = (searchParams.get('range') as SummaryRange) || '30d'; + const range: SummaryRange = RANGE_LOOKBACK[rangeParam] !== undefined ? rangeParam : '30d'; + const limitParam = parseInt(searchParams.get('limit') || '1000', 10); + const limit = Math.min(Math.max(limitParam, 100), 2000); + const now = Date.now(); + const lookback = RANGE_LOOKBACK[range]; + const startTime = lookback ? now - lookback : undefined; + + const records = await getIncomeHistory(credentials, { + incomeType: 'REALIZED_PNL', + startTime, + endTime: now, + limit, + }); + + if (!records || records.length === 0) { + return NextResponse.json({ + symbols: [], + range, + generatedAt: Date.now(), + }); + } + + const activeSymbols = new Set(Object.keys(config.symbols || {})); + const summaryMap = new Map(); + + const upsertSummary = (record: IncomeRecord) => { + const symbol = record.symbol; + if (!symbol) return; + if (activeSymbols.size > 0 && !activeSymbols.has(symbol)) return; + + const amount = parseFloat(record.income || '0'); + if (!summaryMap.has(symbol)) { + summaryMap.set(symbol, { + symbol, + realizedPnl: 0, + tradeCount: 0, + wins: 0, + losses: 0, + winRate: 0, + avgWin: 0, + avgLoss: 0, + largestWin: 0, + largestLoss: 0, + lastTradeTime: null, + totalWin: 0, + totalLoss: 0, + }); + } + + const entry = summaryMap.get(symbol)!; + entry.realizedPnl += amount; + entry.tradeCount += 1; + entry.lastTradeTime = entry.lastTradeTime ? Math.max(entry.lastTradeTime, record.time) : record.time; + + if (amount > 0) { + entry.wins += 1; + entry.totalWin += amount; + entry.largestWin = Math.max(entry.largestWin, amount); + } else if (amount < 0) { + const lossAbs = Math.abs(amount); + entry.losses += 1; + entry.totalLoss += lossAbs; + entry.largestLoss = Math.max(entry.largestLoss, lossAbs); + } + }; + + records.forEach(upsertSummary); + + const summaries: SymbolSummary[] = Array.from(summaryMap.values()).map(entry => { + const { totalWin, totalLoss, ...rest } = entry; + const winRate = rest.tradeCount > 0 ? (rest.wins / rest.tradeCount) * 100 : 0; + return { + ...rest, + winRate, + avgWin: rest.wins > 0 ? totalWin / rest.wins : 0, + avgLoss: rest.losses > 0 ? -(totalLoss / rest.losses) : 0, + }; + }).sort((a, b) => b.realizedPnl - a.realizedPnl); + + const best = summaries.reduce((acc, item) => { + if (!acc || item.realizedPnl > acc.realizedPnl) { + return item; + } + return acc; + }, null); + + const worst = summaries.reduce((acc, item) => { + if (!acc || item.realizedPnl < acc.realizedPnl) { + return item; + } + return acc; + }, null); + + return NextResponse.json({ + symbols: summaries, + best, + worst, + range, + generatedAt: Date.now(), + }); + } catch (error) { + console.error('[Realized PnL Summary] Failed to generate summary:', error); + return NextResponse.json({ + symbols: [], + range: '30d', + generatedAt: Date.now(), + error: error instanceof Error ? error.message : 'Unknown error', + }, { status: 500 }); + } +} diff --git a/src/app/page.tsx b/src/app/page.tsx index d13f28c..6bcafe9 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -16,6 +16,7 @@ import MinimalBotStatus from '@/components/MinimalBotStatus'; import LiquidationSidebar from '@/components/LiquidationSidebar'; import PositionTable from '@/components/PositionTable'; import PnLChart from '@/components/PnLChart'; +import TokenRealizedPnLSummary from '@/components/TokenRealizedPnLSummary'; import PerformanceCardInline from '@/components/PerformanceCardInline'; import RecentOrdersTable from '@/components/RecentOrdersTable'; import { TradeSizeWarningModal } from '@/components/TradeSizeWarningModal'; @@ -387,8 +388,11 @@ export default function DashboardPage() {
- {/* PnL Chart */} - + {/* PnL & Realized Summary */} +
+ + +
{/* Positions Table */} ('30d'); + const [summaries, setSummaries] = useState([]); + const [best, setBest] = useState(null); + const [worst, setWorst] = useState(null); + const [generatedAt, setGeneratedAt] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [isRefreshing, setIsRefreshing] = useState(false); + const [error, setError] = useState(null); + + const fetchSummary = useCallback(async (opts?: { refresh?: boolean }) => { + try { + if (opts?.refresh) { + setIsRefreshing(true); + } else { + setIsLoading(true); + } + setError(null); + + const response = await fetch(`/api/pnl/realized-summary?range=${range}`); + if (!response.ok) { + throw new Error(`Request failed with status ${response.status}`); + } + const data: SummaryResponse = await response.json(); + setSummaries(data.symbols || []); + setBest(data.best || null); + setWorst(data.worst || null); + setGeneratedAt(data.generatedAt || Date.now()); + } catch (err) { + const message = err instanceof Error ? err.message : 'Failed to load summary'; + setError(message); + setSummaries([]); + setBest(null); + setWorst(null); + setGeneratedAt(null); + } finally { + setIsLoading(false); + setIsRefreshing(false); + } + }, [range]); + + useEffect(() => { + fetchSummary(); + }, [fetchSummary]); + + useEffect(() => { + const handler = (message: any) => { + if (message?.type === 'order_update' || message?.type === 'ORDER_TRADE_UPDATE') { + const payload = message?.data?.o || message?.data; + const status = payload?.X; + const realized = payload?.rp; + if (status === 'FILLED' && realized !== undefined) { + fetchSummary({ refresh: true }); + } + } + }; + + const cleanup = websocketService.addMessageHandler(handler); + return () => { + cleanup(); + }; + }, [fetchSummary]); + + const formattedGeneratedAt = useMemo(() => { + if (!generatedAt) return null; + return new Date(generatedAt).toLocaleTimeString(); + }, [generatedAt]); + + const topSummaries = useMemo(() => summaries.slice(0, 10), [summaries]); + + const formatCurrency = (value: number) => { + if (!Number.isFinite(value)) return '$0.00'; + const formatter = new Intl.NumberFormat('en-US', { + style: 'currency', + currency: 'USD', + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + return formatter.format(value); + }; + + const formatPercent = (value: number) => { + if (!Number.isFinite(value)) return '0%'; + return `${value >= 0 ? '' : '-'}${Math.abs(value).toFixed(1)}%`; + }; + + return ( + + +
+ + Token Realized PnL + {isRefreshing && } + +
+ {best && ( + + + Best: {best.symbol.replace('USDT', '')} {formatCurrency(best.realizedPnl)} + + )} + {worst && ( + + + Worst: {worst.symbol.replace('USDT', '')} {formatCurrency(worst.realizedPnl)} + + )} + {formattedGeneratedAt && ( + Updated {formattedGeneratedAt} + )} +
+
+
+ + +
+ + + {isLoading ? ( +
+ {Array.from({ length: 5 }).map((_, index) => ( + + ))} +
+ ) : error ? ( +
+ {error} +
+ ) : summaries.length === 0 ? ( +
+ No realized trades found for this range. +
+ ) : ( +
+ + + + Token + Trades + Win Rate + Avg Win + Avg Loss + Realized PnL + + + + {topSummaries.map(summary => { + const isPositive = summary.realizedPnl >= 0; + return ( + + + {summary.symbol.replace('USDT', '')} + + + {summary.tradeCount} + + ({summary.wins}/{summary.losses}) + + + + {formatPercent(summary.winRate)} + + + {formatCurrency(summary.avgWin)} + + + {formatCurrency(summary.avgLoss)} + + + {formatCurrency(summary.realizedPnl)} + + + ); + })} + +
+
+ )} + + + ); +} diff --git a/src/lib/api/orders.ts b/src/lib/api/orders.ts index 2e62671..99f3cff 100644 --- a/src/lib/api/orders.ts +++ b/src/lib/api/orders.ts @@ -4,6 +4,7 @@ import { buildSignedForm, buildSignedQuery } from './auth'; import { getRateLimitedAxios } from './requestInterceptor'; import { symbolPrecision } from '../utils/symbolPrecision'; import { getMarkPrice } from './market'; +import { normalizeLeverage } from '../utils/leverage'; const BASE_URL = 'https://fapi.asterdex.com'; @@ -194,9 +195,15 @@ export async function getAllOrders(symbol: string, credentials: ApiCredentials, // Change leverage export async function setLeverage(symbol: string, leverage: number, credentials: ApiCredentials): Promise { + const normalizedLeverage = normalizeLeverage(leverage); + + if (normalizedLeverage !== leverage) { + console.warn(`[Orders] Normalizing leverage for ${symbol} from ${leverage}x to ${normalizedLeverage}x (exchange requires whole numbers).`); + } + const params = { symbol, - leverage, + leverage: normalizedLeverage, }; const formData = buildSignedForm(params, credentials); diff --git a/src/lib/bot/hunter.ts b/src/lib/bot/hunter.ts index 71f4e70..08d1316 100644 --- a/src/lib/bot/hunter.ts +++ b/src/lib/bot/hunter.ts @@ -11,6 +11,7 @@ import { vwapService } from '../services/vwapService'; import { vwapStreamer } from '../services/vwapStreamer'; import { thresholdMonitor } from '../services/thresholdMonitor'; import { symbolPrecision } from '../utils/symbolPrecision'; +import { normalizeLeverage } from '../utils/leverage'; import { parseExchangeError, NotionalError, @@ -35,6 +36,7 @@ export class Hunter extends EventEmitter { private cleanupInterval: NodeJS.Timeout | null = null; // Periodic cleanup timer private syncInterval: NodeJS.Timeout | null = null; // Position mode sync timer private lastModeSync: number = Date.now(); // Track last mode sync time + private leverageNormalizationCache: Map = new Map(); constructor(config: Config, isHedgeMode: boolean = false) { super(); @@ -665,6 +667,7 @@ export class Hunter extends EventEmitter { let notionalUSDT: number | undefined; // Don't initialize to 0 - use undefined let tradeSizeUSDT: number = symbolConfig.tradeSize; // Default to general tradeSize let order: any; // Declare order variable for error handling + const leverage = this.getNormalizedLeverage(symbol, symbolConfig.leverage ?? 1); try { // Check position limits before placing trade @@ -705,13 +708,13 @@ export class Hunter extends EventEmitter { } if (this.config.global.paperMode) { - console.log(`Hunter: PAPER MODE - Would place ${side} order for ${symbol}, quantity: ${symbolConfig.tradeSize}, leverage: ${symbolConfig.leverage}`); + console.log(`Hunter: PAPER MODE - Would place ${side} order for ${symbol}, quantity: ${symbolConfig.tradeSize}, leverage: ${leverage}`); this.emit('positionOpened', { symbol, side, quantity: symbolConfig.tradeSize, price: entryPrice, - leverage: symbolConfig.leverage, + leverage, paperMode: true }); return; @@ -780,7 +783,7 @@ export class Hunter extends EventEmitter { ? (symbolConfig.longTradeSize ?? symbolConfig.tradeSize) : (symbolConfig.shortTradeSize ?? symbolConfig.tradeSize); - notionalUSDT = tradeSizeUSDT * symbolConfig.leverage; + notionalUSDT = tradeSizeUSDT * leverage; // Ensure we meet minimum notional requirement if (notionalUSDT < minNotional) { @@ -799,15 +802,15 @@ export class Hunter extends EventEmitter { if (quantity === 0 || quantity < minQty) { // Calculate what the minimum trade size should be - const minNotionalForMargin = minNotional / symbolConfig.leverage; - const minQtyForMargin = (minQty * currentPrice) / symbolConfig.leverage; + const minNotionalForMargin = minNotional / leverage; + const minQtyForMargin = (minQty * currentPrice) / leverage; const recommendedTradeSize = Math.max(minNotionalForMargin, minQtyForMargin) * 1.3; // 30% buffer console.error(`Hunter: Trade size too small for ${symbol} - quantity rounds to zero or below minimum`); console.error(` Current trade size: ${tradeSizeUSDT} USDT`); console.error(` Calculated quantity: ${calculatedQuantity.toFixed(8)} -> ${quantity} (after formatting)`); console.error(` Minimum quantity: ${minQty}`); - console.error(` Minimum notional: ${minNotional} USDT (${minNotionalForMargin.toFixed(2)} USDT at ${symbolConfig.leverage}x leverage)`); + console.error(` Minimum notional: ${minNotional} USDT (${minNotionalForMargin.toFixed(2)} USDT at ${leverage}x leverage)`); console.error(` RECOMMENDED: Set trade size to at least ${recommendedTradeSize.toFixed(2)} USDT`); // Broadcast error to UI @@ -825,7 +828,7 @@ export class Hunter extends EventEmitter { formattedQuantity: quantity, minQuantity: minQty, currentPrice: currentPrice, - leverage: symbolConfig.leverage + leverage: leverage } } ); @@ -852,9 +855,9 @@ export class Hunter extends EventEmitter { } // Set leverage if needed - await setLeverage(symbol, symbolConfig.leverage, this.config.api); + await setLeverage(symbol, leverage, this.config.api); - console.log(`Hunter: Calculated quantity for ${symbol}: margin=${tradeSizeUSDT} USDT (${side === 'BUY' ? 'long' : 'short'}), leverage=${symbolConfig.leverage}x, price=${currentPrice}, notional=${notionalUSDT} USDT, quantity=${quantity}`); + console.log(`Hunter: Calculated quantity for ${symbol}: margin=${tradeSizeUSDT} USDT (${side === 'BUY' ? 'long' : 'short'}), leverage=${leverage}x, price=${currentPrice}, notional=${notionalUSDT} USDT, quantity=${quantity}`); // Quick sanity check - ensure our mode is still in sync (if last sync was over 1 minute ago) if (Date.now() - this.lastModeSync > 60000) { @@ -998,7 +1001,7 @@ export class Hunter extends EventEmitter { quantity, price: orderType === 'LIMIT' ? orderPrice : entryPrice, orderId: order.orderId, - leverage: symbolConfig.leverage, + leverage, orderType, paperMode: false }); @@ -1031,7 +1034,7 @@ export class Hunter extends EventEmitter { symbol, quantity: quantity || 0, // Use actual quantity if calculated, otherwise 0 price: currentPrice, - leverage: symbolConfig.leverage, + leverage, positionSide: getPositionSide(this.isHedgeMode, side) }); @@ -1044,7 +1047,7 @@ export class Hunter extends EventEmitter { side, quantity: quantity || 0, // Use actual quantity if calculated, otherwise 0 price: currentPrice, - leverage: symbolConfig.leverage, + leverage, tradeSizeUSDT, notionalUSDT: notionalUSDT || 0, // Use actual notional if calculated, otherwise 0 errorCode: tradingError.code, @@ -1214,7 +1217,7 @@ export class Hunter extends EventEmitter { let fallbackPositionSide: 'BOTH' | 'LONG' | 'SHORT' = 'BOTH'; try { - await setLeverage(symbol, symbolConfig.leverage, this.config.api); + await setLeverage(symbol, leverage, this.config.api); // Fetch symbol info for precision and filters const fallbackSymbolInfo = await getSymbolFilters(symbol); @@ -1235,7 +1238,7 @@ export class Hunter extends EventEmitter { fallbackPrice = symbolPrecision.formatPrice(symbol, rawFallbackPrice); // Calculate quantity for fallback order - let fallbackNotionalUSDT = symbolConfig.tradeSize * symbolConfig.leverage; + let fallbackNotionalUSDT = symbolConfig.tradeSize * leverage; // Ensure we meet minimum notional requirement if (fallbackNotionalUSDT < fallbackMinNotional) { @@ -1249,7 +1252,7 @@ export class Hunter extends EventEmitter { // Always use symbolPrecision formatting (which now has defaults) fallbackQuantity = symbolPrecision.formatQuantity(symbol, rawFallbackQuantity); - console.log(`Hunter: Fallback calculation for ${symbol}: margin=${symbolConfig.tradeSize} USDT, leverage=${symbolConfig.leverage}x, price=${fallbackPrice}, notional=${fallbackNotionalUSDT} USDT, quantity=${fallbackQuantity}`); + console.log(`Hunter: Fallback calculation for ${symbol}: margin=${symbolConfig.tradeSize} USDT, leverage=${leverage}x, price=${fallbackPrice}, notional=${fallbackNotionalUSDT} USDT, quantity=${fallbackQuantity}`); fallbackPositionSide = getPositionSide(this.isHedgeMode, side) as 'BOTH' | 'LONG' | 'SHORT'; console.log(`Hunter: Using position mode: ${this.isHedgeMode ? 'HEDGE' : 'ONE-WAY'}, side: ${side}, positionSide: ${fallbackPositionSide}`); @@ -1291,7 +1294,7 @@ export class Hunter extends EventEmitter { quantity: fallbackQuantity, price: entryPrice, orderId: fallbackOrder.orderId, - leverage: symbolConfig.leverage, + leverage, orderType: 'MARKET', paperMode: false }); @@ -1308,7 +1311,7 @@ export class Hunter extends EventEmitter { symbol, quantity: fallbackQuantity, price: fallbackPrice, - leverage: symbolConfig.leverage, + leverage, positionSide: fallbackPositionSide }); @@ -1321,7 +1324,7 @@ export class Hunter extends EventEmitter { side, quantity: fallbackQuantity, price: fallbackPrice, - leverage: symbolConfig.leverage, + leverage, tradeSizeUSDT, errorCode: fallbackTradingError.code, errorType: fallbackTradingError.constructor.name, @@ -1448,4 +1451,16 @@ export class Hunter extends EventEmitter { // Start generating events after 2 seconds setTimeout(generateEvent, 2000); } + + private getNormalizedLeverage(symbol: string, configuredLeverage: number): number { + const normalized = normalizeLeverage(configuredLeverage); + if (normalized !== configuredLeverage) { + const lastLogged = this.leverageNormalizationCache.get(symbol); + if (lastLogged !== normalized) { + console.warn(`Hunter: Normalizing leverage for ${symbol} from ${configuredLeverage}x to ${normalized}x (using whole-number leverage).`); + this.leverageNormalizationCache.set(symbol, normalized); + } + } + return normalized; + } } diff --git a/src/lib/utils/leverage.ts b/src/lib/utils/leverage.ts new file mode 100644 index 0000000..7e57b4e --- /dev/null +++ b/src/lib/utils/leverage.ts @@ -0,0 +1,13 @@ +const MIN_LEVERAGE = 1; +const MAX_LEVERAGE = 125; + +export function normalizeLeverage(rawValue: number): number { + if (!Number.isFinite(rawValue)) { + return MIN_LEVERAGE; + } + + const rounded = Math.round(rawValue); + if (rounded < MIN_LEVERAGE) return MIN_LEVERAGE; + if (rounded > MAX_LEVERAGE) return MAX_LEVERAGE; + return rounded; +} diff --git a/src/lib/validation/tradeSizeValidator.ts b/src/lib/validation/tradeSizeValidator.ts index 07b50d1..8b780ac 100644 --- a/src/lib/validation/tradeSizeValidator.ts +++ b/src/lib/validation/tradeSizeValidator.ts @@ -1,6 +1,7 @@ import { Config } from '../types'; import { getExchangeInfo } from '../api/market'; import axios from 'axios'; +import { normalizeLeverage } from '../utils/leverage'; export interface TradeSizeValidationResult { valid: boolean; @@ -51,7 +52,7 @@ export async function validateAllTradeSizes(config: Config): Promise