diff --git a/FORMATIONS_PLAN.md b/FORMATIONS_PLAN.md new file mode 100644 index 0000000..7ef5b46 --- /dev/null +++ b/FORMATIONS_PLAN.md @@ -0,0 +1,1057 @@ +# Chart Formations Feature Implementation Plan + +## Summary + +Add chart formations feature with: +- **9 pattern types**: Line, Channel, Triangle, H&S, Double Bottom/Top, Adam & Eve, Triple Bottom/Top +- **Interactive drawing**: Click-to-place shape, drag anchors to adjust (SVG overlay) +- **HUD panel**: Manage formations like indicators/signals +- **Strategy integration**: Reference formations in Blockly (by tbl_key, not name) +- **Auto-targets**: Calculate and draw projected target levels +- **Infinite extension**: Lines extrapolate beyond drawn endpoints +- **Ownership model**: Users own formations; public strategies use creator's formations +- **Scope**: Formations tied to exchange/market/timeframe + +**Rendering approach**: SVG overlay with requestAnimationFrame sync (NOT Lightweight Charts line series - avoids infinite loops with bound charts). + +## Supported Patterns + +| Pattern | Points | Description | +|---------|--------|-------------| +| Support/Resistance | 2 | Single line (horizontal or diagonal) | +| Channel | 3 | Two parallel trendlines | +| Triangle | 3 | Three connected lines (ascending/descending/symmetrical) | +| Head & Shoulders | 5 | Reversal pattern with neckline + 3 peaks | +| Double Bottom (W) | 3 | Two bottoms with middle peak (bullish reversal) | +| Double Top (M) | 3 | Two tops with middle trough (bearish reversal) | +| Adam and Eve | 3 | Sharp V bottom + rounded U bottom variant | +| Triple Bottom | 5 | Three bottoms with two peaks (strong bullish reversal) | +| Triple Top | 5 | Three tops with two troughs (strong bearish reversal) | + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Formations HUD Panel │ +│ (Draw tools, formation cards, CRUD) │ +├─────────────────────────────────────────────────────────────────┤ +│ FormationOverlay │ +│ (SVG layer positioned over chart, draggable anchors) │ +├─────────────────────────────────────────────────────────────────┤ +│ Formations.py (Backend) │ +│ (CRUD, value calculation, target projection, scope) │ +├─────────────────────────────────────────────────────────────────┤ +│ SQLite (formations table) │ +│ (id, user_id, name, type, scope, lines_json) │ +└─────────────────────────────────────────────────────────────────┘ + +Strategy Integration: + Blockly Block → PythonGenerator → process_formation(tbl_key, property, timestamp) → price + +Rendering Approach: + - SVG overlay sits absolutely positioned over the Lightweight Charts canvas + - Formations drawn as SVG elements (line, polygon, circle anchors) + - Chart sync via polling (requestAnimationFrame), NOT event subscriptions + - Avoids infinite loops from subscribeVisibleTimeRangeChange +``` + +## Key Design Decisions + +1. **Ownership**: Users own their formations. Public strategies reference creator's formations (subscribers can't see/edit them but the strategy uses them). Uses `formation_owner_id` pattern (parallel to `indicator_owner_id`). + +2. **Line Extension**: All lines extend infinitely (extrapolate beyond drawn endpoints). MVP has no extension options - all lines are infinite. + +3. **Target Projection**: Auto-calculate target levels as horizontal lines (e.g., H&S target = neckline - pattern_height). Added in Phase B, not MVP. + +4. **Scope**: Uses `exchange`, `market`, `timeframe` (matching existing patterns - `market` not `symbol`). + +5. **References**: Blocks reference formations by `tbl_key` (stable UUID) not name (avoids collisions). + +6. **Timestamps**: All timestamps in seconds UTC. Backtest uses bar timestamp, not wall clock. Out-of-range returns extrapolated value (infinite lines). + +7. **MVP Scope**: Ship Line + Channel first. Complex patterns (Triangle, H&S, etc.) in Phase B. + +--- + +## Phase 1: Backend Foundation + +### 1.1 Database Schema + +**In `src/Formations.py`** - Use `_ensure_table_exists()` pattern (like Signals/Trades): + +```python +class Formations: + TABLE_NAME = 'formations' + + def __init__(self, data_cache: DataCache, database: Database): + self.data_cache = data_cache + self.database = database + self._ensure_table_exists() + self._init_cache() + + def _ensure_table_exists(self): + """Create formations table if it doesn't exist (repo pattern).""" + self.database.execute_sql(''' + CREATE TABLE IF NOT EXISTS formations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tbl_key TEXT UNIQUE NOT NULL, + user_id INTEGER NOT NULL, + name TEXT NOT NULL, + formation_type TEXT NOT NULL, + exchange TEXT NOT NULL, + market TEXT NOT NULL, + timeframe TEXT NOT NULL, + lines_json TEXT NOT NULL, + color TEXT DEFAULT '#667eea', + visible INTEGER DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id), + UNIQUE(user_id, name, exchange, market, timeframe) + ) + ''') + self.database.execute_sql(''' + CREATE INDEX IF NOT EXISTS idx_formations_scope + ON formations(user_id, exchange, market, timeframe) + ''') + + def _init_cache(self): + """Initialize formations cache using DataCache pattern.""" + self.data_cache.create_cache('formations', {}) +``` + +**Lines JSON Structure (MVP - infinite lines only):** +```json +{ + "lines": [ + {"point1": {"time": 1709900000, "price": 65000.0}, + "point2": {"time": 1710500000, "price": 66000.0}} + ], + "targets": [ + {"price": 68000.0, "label": "target_1"} + ] +} +``` + +**Note**: MVP uses infinite line extension only. No `extend` field needed. + +### 1.2 Backend Module + +**New file:** `src/Formations.py` + +```python +class Formations: + def __init__(self, database: Database): + self.database = database + + # CRUD operations + def create(self, user_id: int, data: dict) -> dict + def update(self, user_id: int, data: dict) -> dict + def delete(self, user_id: int, tbl_key: str) -> dict + + # Queries - always constrained by user_id + def get_for_scope(self, user_id: int, exchange: str, market: str, timeframe: str) -> list + def get_by_tbl_key(self, user_id: int, tbl_key: str) -> dict | None + def get_by_tbl_key_for_strategy(self, tbl_key: str, owner_user_id: int) -> dict | None + """For strategy execution - uses strategy owner's formations""" + + # Value calculation + def calculate_line_value(self, line: dict, timestamp: int) -> float + def calculate_target_value(self, formation: dict, target_name: str) -> float + def get_property_value(self, formation: dict, property: str, timestamp: int) -> float +``` + +**Value calculation (core algorithm - infinite extension):** +```python +def calculate_line_value(self, line: dict, timestamp: int) -> float: + """Calculate price at timestamp using linear interpolation/extrapolation. + + Args: + line: {"point1": {"time": int, "price": float}, "point2": {...}} + timestamp: Unix timestamp in seconds UTC + + Returns: + Extrapolated price value (works for any timestamp, past or future) + + Edge case: t1 == t2 is a vertical line (same time, different prices). + This shouldn't happen in normal drawing but we handle it gracefully. + """ + t1, p1 = line['point1']['time'], line['point1']['price'] + t2, p2 = line['point2']['time'], line['point2']['price'] + + if t1 == t2: + # Vertical line in time (same timestamp) - invalid for price lookup + # Return average price as fallback + logger.warning(f"Vertical line detected (t1==t2={t1}), returning average price") + return (p1 + p2) / 2 + + slope = (p2 - p1) / (t2 - t1) + return p1 + slope * (timestamp - t1) + +def calculate_targets(self, formation_type: str, lines: list, points: dict) -> list: + """Auto-calculate target levels based on formation type. + + Returns list of horizontal target lines: + - H&S: neckline - pattern_height + - Double bottom: neckline + pattern_height + - Triangle: apex projection + """ + targets = [] + # Pattern-specific target calculation logic + return targets +``` + +### 1.3 Socket Events + +**File:** `src/BrighterTrades.py` - Add to `process_incoming_message()` at line ~1548 + +Keep ALL socket handling in `BrighterTrades.process_incoming_message`, not app.py. Use existing reply envelope shape (`reply` + `data`): + +```python +# In process_incoming_message(): +elif message_type == 'request_formations': + formations = self.formations.get_for_scope(user_id, data) + return {'reply': 'formations', 'data': {'formations': formations}} + +elif message_type == 'new_formation': + result = self.formations.create(user_id, data) + return {'reply': 'formation_created', 'data': result} + +elif message_type == 'edit_formation': + result = self.formations.update(user_id, data) + return {'reply': 'formation_updated', 'data': result} + +elif message_type == 'delete_formation': + result = self.formations.delete(user_id, data['tbl_key']) + return {'reply': 'formation_deleted', 'data': result} +``` + +**Frontend handler** (in `communication.js`, follows existing pattern at line ~88): +```javascript +// Socket response handler +socket.on('message', (msg) => { + if (msg.reply === 'formations') { + UI.formations.handleFormations(msg.data); + } else if (msg.reply === 'formation_created') { + UI.formations.handleFormationCreated(msg.data); + } + // ... etc +}); +``` + +--- + +## Phase 2: Frontend HUD Panel + +### 2.1 Template + +**New file:** `src/templates/formations_hud.html` + +```html + +
+

Draw Formation

+
+ + +
+ + + +
+

Formations

+
+
+``` + +### 2.2 JavaScript Controller + +**New file:** `src/static/formations.js` + +Follow the three-class pattern from `signals.js`: + +| Class | Purpose | +|-------|---------| +| `FormationsUIManager` | DOM rendering, card creation | +| `FormationsDataManager` | In-memory formations array | +| `Formations` | Socket handlers, coordinates UI + Data | + +**New file:** `src/static/formation_overlay.js` + +| Class | Purpose | +|-------|---------| +| `FormationOverlay` | SVG rendering, anchor dragging, chart sync | + +### 2.3 Integration Points + +**Modify:** `src/templates/control_panel.html` +```html + +{% include "formations_hud.html" %} +``` + +**Modify:** `src/static/general.js` +```javascript +this.formations = new Formations(this); +// After charts init: +this.formations.initOverlay('chart1', this.charts.chart_1); +``` + +--- + +## Phase 3: Chart Drawing (SVG Overlay Approach) + +### 3.1 Why SVG Overlay Instead of addLineSeries + +**Problem with addLineSeries approach:** +- Lightweight Charts `subscribeVisibleTimeRangeChange` triggers infinite loops when multiple charts are bound together +- Adding guard flags (`_syncing`) helped but still caused page freezing after a few seconds +- Line series participate in chart scaling, causing unwanted price scale changes +- No native support for draggable anchor points + +**SVG Overlay benefits:** +- Completely decoupled from chart internals - no event subscription loops +- Full control over anchor points and drag behavior +- Lines don't affect price scale calculations +- Simpler mental model: SVG is a transparent layer on top of chart + +### 3.2 FormationOverlay Class + +**New file:** `src/static/formation_overlay.js` + +```javascript +class FormationOverlay { + constructor(chartContainerId, chart) { + this.chart = chart; + this.container = document.getElementById(chartContainerId); + this.svg = null; + this.formations = new Map(); // tbl_key -> {group, anchors, lines} + this.selectedFormation = null; + this.draggingAnchor = null; + + this._createSvgLayer(); + this._startSyncLoop(); // NOT event subscription + } + + _createSvgLayer() { + // Create SVG absolutely positioned over chart canvas + this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + this.svg.style.cssText = ` + position: absolute; + top: 0; left: 0; + width: 100%; height: 100%; + pointer-events: none; // Let clicks pass through to chart + z-index: 10; + `; + this.container.style.position = 'relative'; + this.container.appendChild(this.svg); + } + + _startSyncLoop() { + // CRITICAL: Use requestAnimationFrame polling, NOT subscribeVisibleTimeRangeChange + // This avoids the infinite loop problem with bound charts + const sync = () => { + this._updateAllPositions(); + this._animationFrameId = requestAnimationFrame(sync); + }; + this._animationFrameId = requestAnimationFrame(sync); + } + + stopSyncLoop() { + if (this._animationFrameId) { + cancelAnimationFrame(this._animationFrameId); + } + } + + _updateAllPositions() { + // Reposition all SVG elements based on current chart coordinates + for (const [tblKey, formation] of this.formations) { + this._updateFormationPosition(tblKey, formation); + } + } + + _updateFormationPosition(tblKey, formation) { + const lines = formation.data.lines; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const p1 = this._chartToPixel(line.point1.time, line.point1.price); + const p2 = this._chartToPixel(line.point2.time, line.point2.price); + + // Update line element + const lineEl = formation.lineElements[i]; + if (lineEl && p1 && p2) { + lineEl.setAttribute('x1', p1.x); + lineEl.setAttribute('y1', p1.y); + lineEl.setAttribute('x2', p2.x); + lineEl.setAttribute('y2', p2.y); + } + + // Update anchor elements + const anchor1 = formation.anchorElements[i * 2]; + const anchor2 = formation.anchorElements[i * 2 + 1]; + if (anchor1 && p1) { + anchor1.setAttribute('cx', p1.x); + anchor1.setAttribute('cy', p1.y); + } + if (anchor2 && p2) { + anchor2.setAttribute('cx', p2.x); + anchor2.setAttribute('cy', p2.y); + } + } + } + + _chartToPixel(time, price) { + // Convert chart coordinates to SVG pixel coordinates + const timeScale = this.chart.timeScale(); + const priceScale = this.chart.priceScale('right'); + + const x = timeScale.timeToCoordinate(time); + const y = priceScale.priceToCoordinate(price); + + if (x === null || y === null) return null; + return { x, y }; + } + + _pixelToChart(x, y) { + // Convert SVG pixel coordinates to chart coordinates + const timeScale = this.chart.timeScale(); + const priceScale = this.chart.priceScale('right'); + + const time = timeScale.coordinateToTime(x); + const price = priceScale.coordinateToPrice(y); + + return { time, price }; + } +} +``` + +### 3.3 Click-to-Place UX + +**User interaction flow:** +1. User clicks "Line" button in HUD panel +2. A default line shape appears at chart center (not following mouse - simpler) +3. User clicks chart to place the shape +4. Shape becomes editable with visible anchor points +5. User drags anchors to adjust position +6. User clicks "Save" or hits Enter to persist + +```javascript +// In FormationOverlay +startDrawing(type) { + this.drawingMode = type; + + // Create default shape at chart center + const center = this._getChartCenter(); + const defaultPoints = this._getDefaultPointsForType(type, center); + + // Create temporary formation (not saved yet) + this.tempFormation = this._createFormationElements( + 'temp_' + Date.now(), + defaultPoints, + { color: '#667eea', isTemp: true } + ); + + // Enable anchor dragging + this._enableAnchors(this.tempFormation); + + // Show save/cancel buttons + this._showDrawingControls(); +} + +_getDefaultPointsForType(type, center) { + const offset = 50; // pixels + + switch (type) { + case 'support_resistance': + return { + lines: [{ + point1: this._pixelToChart(center.x - offset, center.y), + point2: this._pixelToChart(center.x + offset, center.y) + }] + }; + case 'channel': + return { + lines: [ + { + point1: this._pixelToChart(center.x - offset, center.y - 20), + point2: this._pixelToChart(center.x + offset, center.y - 20) + }, + { + point1: this._pixelToChart(center.x - offset, center.y + 20), + point2: this._pixelToChart(center.x + offset, center.y + 20) + } + ] + }; + // ... other types + } +} + +completeDrawing() { + if (!this.tempFormation) return; + + // Convert temp formation to permanent + const formationData = { + formation_type: this.drawingMode, + lines_json: JSON.stringify(this.tempFormation.data), + color: '#667eea' + }; + + // Prompt for name + const name = prompt('Formation name:'); + if (!name) { + this.cancelDrawing(); + return; + } + + formationData.name = name; + + // Save via socket + this.onSave(formationData); // Callback to formations.js + + // Cleanup temp + this._removeFormationElements(this.tempFormation); + this.tempFormation = null; + this.drawingMode = null; +} +``` + +### 3.4 Draggable Anchors + +```javascript +_createAnchor(x, y, lineIndex, pointIndex, formation) { + const anchor = document.createElementNS('http://www.w3.org/2000/svg', 'circle'); + anchor.setAttribute('cx', x); + anchor.setAttribute('cy', y); + anchor.setAttribute('r', 6); + anchor.setAttribute('fill', '#667eea'); + anchor.setAttribute('stroke', '#fff'); + anchor.setAttribute('stroke-width', 2); + anchor.style.cursor = 'grab'; + anchor.style.pointerEvents = 'all'; // Enable interaction + + // Drag handlers + anchor.addEventListener('mousedown', (e) => { + e.stopPropagation(); + this.draggingAnchor = { anchor, lineIndex, pointIndex, formation }; + anchor.style.cursor = 'grabbing'; + }); + + return anchor; +} + +_setupDragListeners() { + document.addEventListener('mousemove', (e) => { + if (!this.draggingAnchor) return; + + const rect = this.svg.getBoundingClientRect(); + const x = e.clientX - rect.left; + const y = e.clientY - rect.top; + + // Update anchor position + const { anchor, lineIndex, pointIndex, formation } = this.draggingAnchor; + anchor.setAttribute('cx', x); + anchor.setAttribute('cy', y); + + // Update formation data + const chartCoords = this._pixelToChart(x, y); + const pointKey = pointIndex === 0 ? 'point1' : 'point2'; + formation.data.lines[lineIndex][pointKey] = { + time: chartCoords.time, + price: chartCoords.price + }; + + // Update line element + this._updateFormationPosition(formation.tblKey, formation); + }); + + document.addEventListener('mouseup', () => { + if (this.draggingAnchor) { + this.draggingAnchor.anchor.style.cursor = 'grab'; + this.draggingAnchor = null; + + // If editing existing formation, auto-save + if (this.selectedFormation && !this.selectedFormation.isTemp) { + this._autoSaveFormation(this.selectedFormation); + } + } + }); +} +``` + +### 3.5 Drawing Interaction Summary + +| Formation Type | Default Shape | Anchor Points | +|----------------|---------------|---------------| +| Support/Resistance | Horizontal line at center | 2 (endpoints) | +| Channel | Two parallel lines | 4 (2 per line) | +| Triangle | Three connected lines | 3 (vertices) | +| Head & Shoulders | 5-point pattern | 5 (key points) | +| Double Bottom (W) | W-shape | 3 (bottoms + peak) | +| Double Top (M) | M-shape | 3 (tops + trough) | + +**Key UX principles:** +- Shape appears instantly on button click (no multi-click placement) +- Drag anchors to adjust - intuitive and immediate feedback +- Save explicitly via button/Enter +- Cancel via Escape or cancel button + +### 3.6 Lessons Learned (from First Attempt) + +**DO NOT:** +- Use `subscribeVisibleTimeRangeChange` for chart sync - causes infinite loops with bound charts +- Add line series that participate in price scale calculations +- Rely on chart events for positioning updates + +**DO:** +- Use `requestAnimationFrame` polling for smooth, loop-free updates +- Keep SVG layer completely separate from chart internals +- Use `pointer-events: none` on SVG container, `all` only on anchors +- Store chart coordinates (time, price) not pixel coordinates in data + +--- + +## Phase 4: Blockly Integration + +### 4.1 Block Definition + +**New file:** `src/static/blocks/formation_blocks.js` + +Dynamic blocks generated per formation (like signals/indicators): + +```javascript +// Block type: formation_ +// Message: "Formation: {name} {property} at time {timestamp}" +// Output: dynamic_value +// Properties vary by type (all return price at timestamp or constant): +``` + +**Properties by Pattern Type:** + +| Pattern | Properties | Description | +|---------|------------|-------------| +| support_resistance | `line`, `target_1` | Line + auto-target (if applicable) | +| channel | `upper`, `lower`, `midline` | Upper/lower bounds + calculated midline | +| triangle | `side_1`, `side_2`, `side_3`, `apex`, `target_1` | Sides + apex price + breakout target | +| head_shoulders | `neckline`, `left_shoulder`, `head`, `right_shoulder`, `pattern_height`, `target_1` | + auto-calculated target | +| double_bottom | `neckline`, `left_bottom`, `right_bottom`, `pattern_height`, `target_1` | Neckline + bullish target | +| double_top | `neckline`, `left_top`, `right_top`, `pattern_height`, `target_1` | Neckline + bearish target | +| adam_eve | `neckline`, `adam`, `eve`, `pattern_height`, `target_1` | Same as double_bottom | +| triple_bottom | `neckline`, `bottom_1`, `bottom_2`, `bottom_3`, `pattern_height`, `target_1` | Connects peaks + target | +| triple_top | `neckline`, `top_1`, `top_2`, `top_3`, `pattern_height`, `target_1` | Connects troughs + target | + +**Target Calculation:** +- **H&S**: `target_1 = neckline_price - pattern_height` (bearish target) +- **Double Bottom**: `target_1 = neckline_price + pattern_height` (bullish target) +- **Triangle**: `target_1 = apex_price + pattern_height` (breakout direction) +- Targets are drawn as horizontal lines on chart + +**Strategy Usage Examples:** +```python +# Reference by tbl_key (UUID), not name - avoids collisions +# Check if price breaks above double bottom neckline +price > process_formation('f8a3b2c1-...', 'neckline', current_time) + +# Get auto-calculated target level +target = process_formation('f8a3b2c1-...', 'target_1', current_time) + +# Current candle time for live/backtest +process_formation('f8a3b2c1-...', 'line', get_current_candle_time()) +``` + +### 4.2 Python Generator + +**Modify:** `src/PythonGenerator.py` + +```python +def handle_formation(self, node, indent_level): + """Generate process_formation call using tbl_key (stable) not name.""" + fields = node.get('fields', {}) + inputs = node.get('inputs', {}) + tbl_key = fields.get('TBL_KEY') # UUID reference, not name + property = fields.get('PROPERTY', 'line') + timestamp = self.generate_condition_code(inputs.get('TIMESTAMP', {}), indent_level) + + # Track formation usage for dependency resolution + if not hasattr(self, 'formations_used'): + self.formations_used = [] + self.formations_used.append({'tbl_key': tbl_key, 'property': property}) + + return f"process_formation('{tbl_key}', '{property}', {timestamp})" + +def handle_current_candle_time(self, node, indent_level): + return "get_current_candle_time()" +``` + +### 4.4 Blockly Toolbox Refresh + +**Issue**: Dynamic blocks are initialized once (`blocksDefined` flag in `Strategies.js:989`). +New formations won't appear until toolbox rebuild. + +**Solution**: Add formation change listener to trigger toolbox refresh: + +```javascript +// In formations.js - after formation CRUD +handleFormationCreated(data) { + this.dataManager.addFormation(data.formation); + this.uiManager.renderFormations(this.dataManager.formations); + // Trigger toolbox rebuild + UI.strats.workspaceManager.rebuildToolbox(); +} + +// In Strategies.js - add rebuildToolbox method +rebuildToolbox() { + // Clear and rebuild dynamic categories + defineFormationBlocks(); // Re-run block definitions + this.workspace.updateToolbox(document.getElementById('toolbox_advanced')); +} +``` + +### 4.3 Strategy Instance Integration + +**Critical**: Must be wired into ALL strategy instances (paper/live/backtest). + +**Modify:** `src/StrategyInstance.py` (at line ~76, exec context setup) + +```python +class StrategyInstance: + def __init__(self, ..., formations_manager=None, formation_owner_id=None): + # ... existing init + self.formations_manager = formations_manager + # formation_owner_id = strategy creator for subscribed strategies + # Parallel to existing indicator_owner_id pattern + self.formation_owner_id = formation_owner_id or strategy.get('user_id') + + def _build_exec_context(self) -> dict: + """Add process_formation to exec context (line ~76).""" + context = { + # ... existing functions + 'process_formation': self.process_formation, + 'get_current_candle_time': self.get_current_candle_time, + } + return context + + def process_formation(self, tbl_key: str, property: str, timestamp: int = None) -> float: + """Returns price value of formation property at timestamp. + + Uses formation_owner_id (not current user) for subscribed strategies. + Parallel to indicator_owner_id pattern. + """ + if not self.formations_manager: + logger.warning("Formations manager not initialized") + return None + + # Default timestamp: current candle time (not wall clock) + if timestamp is None: + timestamp = self.get_current_candle_time() + + formation = self.formations_manager.get_by_tbl_key_for_strategy( + tbl_key, self.formation_owner_id + ) + if not formation: + return None + + return self.formations_manager.get_property_value(formation, property, timestamp) + + def get_current_candle_time(self) -> int: + """Returns current candle timestamp in seconds UTC. + + IMPORTANT: In backtest, returns bar timestamp (not wall clock). + In live/paper, returns current candle's open time. + """ + return self.current_candle.get('time', int(time.time())) +``` + +**Modify:** `src/Strategies.py` (strategy instance creation, line ~268) + +```python +# When creating strategy instances, inject formations manager + owner_id: +instance = PaperStrategyInstance( + ..., + formations_manager=self.brighter_trades.formations, + formation_owner_id=strategy.get('user_id') # Creator's formations +) +``` + +--- + +## Phase 5: Backtest Support + +**Timestamp Rules:** +- **Backtest**: `get_current_candle_time()` returns the bar's timestamp being processed +- **Live/Paper**: Returns current candle's open time +- **Never use wall clock** for formation lookups in strategy code + +**Modify:** `src/backtest_strategy_instance.py` + +```python +class BacktestStrategyInstance(StrategyInstance): + def __init__(self, ..., formations_manager=None, formation_owner_id=None): + super().__init__( + ..., + formations_manager=formations_manager, + formation_owner_id=formation_owner_id + ) + + def get_current_candle_time(self) -> int: + """In backtest, return the bar timestamp being processed. + + This is critical: backtest must use historical bar time, + not wall clock time, for accurate formation value lookups. + """ + return self.current_bar_time # Set by backtest loop + + # process_formation inherited - uses get_current_candle_time() +``` + +**Modify:** `src/backtesting.py` (line ~854, strategy instance creation) + +```python +# Inject formations manager + owner_id when creating backtest instance +instance = BacktestStrategyInstance( + ..., + formations_manager=brighter_trades.formations, + formation_owner_id=strategy.get('user_id') +) +``` + +--- + +## Phase 6: Chart Scope Lifecycle + +**Issue**: Chart scope changes via form submit/page reload (`price_chart.html:11`), not dynamic events. + +**Solution**: Formation loading tied to page lifecycle: + +```javascript +// In formations.js +initialize(targetId) { + this.uiManager.initUI(targetId); + this.registerSocketHandlers(); + + // Load formations for current chart scope on page load + const chartView = window.bt_data?.chart_view; + if (chartView) { + this.loadFormationsForScope(chartView.exchange, chartView.market, chartView.timeframe); + } +} + +loadFormationsForScope(exchange, market, timeframe) { + this.currentScope = { exchange, market, timeframe }; + this.comms.emit('message', { + type: 'request_formations', + exchange, market, timeframe + }); +} + +// On scope change (if we add dynamic switching later): +onChartScopeChanged(newScope) { + this.drawingManager.clearAllFormations(); + this.loadFormationsForScope(newScope.exchange, newScope.market, newScope.timeframe); +} +``` + +--- + +## Files Summary + +### New Files +| File | Purpose | +|------|---------| +| `src/Formations.py` | Backend CRUD + value calculation | +| `src/static/formations.js` | Frontend controller (UI, Data, Socket handlers) | +| `src/static/formation_overlay.js` | SVG overlay, anchor dragging, chart sync | +| `src/templates/formations_hud.html` | HUD panel template | +| `src/static/blocks/formation_blocks.js` | Blockly block definitions | + +### Modified Files +| File | Changes | +|------|---------| +| `src/Database.py` | Add formations table | +| `src/BrighterTrades.py` | Initialize Formations manager | +| `src/app.py` | Add socket handlers | +| `src/PythonGenerator.py` | Add handle_formation, handle_current_candle_time | +| `src/StrategyInstance.py` | Add process_formation, get_current_candle_time | +| `src/backtest_strategy_instance.py` | Load formations for backtest scope | +| `src/templates/control_panel.html` | Include formations_hud.html | +| `src/static/general.js` | Initialize UI.formations | +| `src/templates/index.html` | Include formations.js + formation_overlay.js | + +--- + +## Verification Plan + +1. **Database**: Run app, verify `formations` table created +2. **CRUD**: Create/edit/delete formation via HUD panel, verify persistence +3. **Drawing**: Draw line on chart using SVG overlay, drag anchors to adjust, verify it renders and persists on page reload. Scroll/zoom chart while formation visible - verify no freezing. +4. **Scope filtering**: Switch exchange/symbol/timeframe, verify formations update +5. **Blockly**: Create strategy referencing formation, verify code generates correctly +6. **Backtest**: Run backtest with formation-based strategy, verify values calculated correctly +7. **Tests**: Add unit tests for `calculate_line_value()` and formation CRUD + +--- + +## Implementation Order + +**Phase A: MVP (ship first - Line + Channel only)** +1. `src/Formations.py` - CRUD, `_ensure_table_exists()`, DataCache, line value calc +2. `src/BrighterTrades.py` - Add to `process_incoming_message()` (~line 1548) +3. `src/templates/formations_hud.html` - HUD panel (Line + Channel buttons only) +4. `src/static/formations.js` - UIManager, DataManager, Controller +5. `src/static/formation_overlay.js` - SVG layer, anchor dragging, requestAnimationFrame sync +6. `src/static/communication.js` - Add reply handlers (~line 88) +7. `src/static/blocks/formation_blocks.js` - Dynamic blocks (tbl_key reference) +8. `src/PythonGenerator.py` - Add `handle_formation`, `handle_current_candle_time` +9. `src/StrategyInstance.py` - Add to exec context (~line 76), inject formations_manager +10. `src/Strategies.py` - Inject formations_manager + formation_owner_id (~line 268) +11. `src/backtest_strategy_instance.py` - Override `get_current_candle_time()` for bar time +12. `src/backtesting.py` - Inject formations_manager (~line 854) +13. Toolbox refresh logic (rebuild on formation create/delete) +14. `tests/test_formations.py` - CRUD, value calc, ownership tests + +**Phase B: Complex Patterns (iterate after MVP ships)** +14. Triangle pattern (3-point drawing) +15. Double Bottom/Top (W/M patterns) +16. Adam and Eve variant +17. Head & Shoulders (5-point) +18. Triple Bottom/Top + +**Phase C: Targets (after patterns stable)** +19. Auto-calculate target levels per pattern type +20. Render targets as horizontal lines on chart +21. Add `target_1` property to Blockly blocks + +--- + +## Tests Required + +**Backend (`tests/test_formations.py`):** +```python +def test_create_formation_unique_constraint(): + """Cannot create duplicate name in same scope""" + +def test_calculate_line_value_interpolation(): + """Value at midpoint timestamp""" + +def test_calculate_line_value_extrapolation(): + """Value beyond endpoints (infinite extension)""" + +def test_get_by_tbl_key_for_strategy_owner(): + """Strategy uses owner's formations, not current user's""" + +def test_calculate_targets_head_shoulders(): + """Target = neckline - pattern_height""" + +def test_calculate_targets_double_bottom(): + """Target = neckline + pattern_height""" +``` + +**Generator (`tests/test_strategy_generation.py`):** +```python +def test_formation_block_generates_process_formation(): + """Block with tbl_key generates correct function call""" +``` + +**Integration (`tests/test_strategy_execution.py`):** +```python +def test_process_formation_in_paper_strategy(): + """Formation value accessible in paper trading""" + +def test_process_formation_in_backtest(): + """Formation value correct at historical timestamp""" + +def test_subscribed_strategy_uses_creator_formations(): + """Subscriber runs strategy with creator's formations""" +``` + +--- + +## Verification (End-to-End Test) + +1. **Start app**: `cd src && python app.py` +2. **Create formation**: + - Open browser to `http://127.0.0.1:5002` + - Navigate to Formations panel + - Click "Line" button - line appears at chart center with visible anchors + - Drag anchors to desired position + - Click "Save" and enter a name + - Verify it appears in panel and persists on page reload + - **Scroll chart** - verify no freezing (critical regression test) + +3. **Use in strategy**: + - Create new strategy in Blockly + - Add Formation block (should see your formation by name, references tbl_key internally) + - Set condition: `if price > formation.line at current_time` + - Save strategy + +4. **Run backtest**: + - Run backtest with the strategy + - Verify formation value calculated correctly at historical timestamps + - Check logs for `process_formation` calls + +5. **Run tests**: `pytest tests/test_formations.py -v` + +--- + +## Appendix: Lessons Learned from First Implementation Attempt + +The first implementation attempt used Lightweight Charts `addLineSeries()` to render formations. This caused page freezing issues. Here's what we learned: + +### What Went Wrong + +1. **Infinite Loop with Bound Charts** + - Used `subscribeVisibleTimeRangeChange()` to sync formation positions when chart scrolled + - BrighterTrading has multiple charts bound together (main chart + indicator charts) + - When one chart scrolled, the sync callback triggered, which updated formation positions, which triggered another scroll event on bound charts, creating an infinite loop + - Adding `_syncing` guard flags reduced but didn't eliminate the problem + +2. **Line Series Side Effects** + - `addLineSeries()` adds data that participates in price scale calculations + - Formation lines affected the auto-scaling of the price axis + - Lines were not truly "overlay" - they were part of chart data + +3. **Incremental Fixes Created Instability** + - Each bug fix introduced new edge cases + - Guard flags, debouncing, and throttling added complexity without solving root cause + - Codebase became fragile and hard to reason about + +### The SVG Overlay Solution + +1. **Complete Separation** + - SVG layer is absolutely positioned over chart canvas + - No interaction with Lightweight Charts internals + - No event subscriptions that could loop + +2. **Polling Instead of Events** + - Use `requestAnimationFrame` to poll chart coordinates + - Smooth 60fps updates without triggering events + - No risk of infinite loops + +3. **Simpler UX Model** + - Click-to-place with instant shape appearance + - Drag anchors to adjust (familiar interaction pattern) + - Clear separation between "drawing" and "adjusting" + +### Key Takeaways + +| Problem | Bad Approach | Good Approach | +|---------|--------------|---------------| +| Chart sync | `subscribeVisibleTimeRangeChange` | `requestAnimationFrame` polling | +| Rendering | `addLineSeries()` | SVG overlay | +| Placement | Multi-click point collection | Shape appears, drag to adjust | +| Event handling | Chart event subscriptions | Minimal DOM events on SVG only | + +### Reference Implementation + +The experimental SVG overlay code is preserved in branch `feature/formations-svg-overlay-wip` for reference. While that implementation still had issues (the page freeze wasn't fully resolved), the overall SVG overlay architecture is sound and should be used as a starting point. diff --git a/src/static/general.js b/src/static/general.js index 5d62a28..830730c 100644 --- a/src/static/general.js +++ b/src/static/general.js @@ -15,7 +15,7 @@ class User_Interface { this.account = new Account(); // Register a callback function for when indicator updates are received from the data object - this.data.registerCallback_i_updates(this.indicators.update); + this.data.registerCallback_i_updates(this.indicators.update.bind(this.indicators)); // Initialize all components after the page has loaded and Blockly is ready this.initializeAll(); diff --git a/src/static/indicators.js b/src/static/indicators.js index 38f3939..d70c08f 100644 --- a/src/static/indicators.js +++ b/src/static/indicators.js @@ -111,8 +111,6 @@ class Indicator { } setLine(lineName, data, value_name) { - console.log('indicators[68]: setLine takes:(lineName, data, value_name)'); - console.log(lineName, data, value_name); let priceValue; @@ -143,6 +141,7 @@ class Indicator { } updateDisplay(name, priceValue, value_name) { + // Try the old element format first (legacy chart-based display) let element = document.getElementById(this.name + '_' + value_name); if (element) { if (typeof priceValue === 'object' && priceValue !== null) { @@ -172,7 +171,36 @@ class Indicator { element.style.height = 'auto'; // Reset height element.style.height = (element.scrollHeight) + 'px'; // Adjust height based on content } else { - console.warn(`Element with ID ${this.name}_${value_name} not found.`); + // Try the new card-based display element + const cardElement = document.getElementById(`indicator_card_value_${this.name}`); + if (cardElement) { + let displayValue = '--'; + if (typeof priceValue === 'object' && priceValue !== null) { + // For object values, get the first numeric value + const values = Object.values(priceValue).filter(v => typeof v === 'number' && !isNaN(v)); + if (values.length > 0) { + displayValue = this._formatDisplayValue(values[0]); + } + } else if (typeof priceValue === 'number' && !isNaN(priceValue)) { + displayValue = this._formatDisplayValue(priceValue); + } + cardElement.textContent = displayValue; + } + // Silently ignore if neither element exists (may be during initialization) + } + } + + /** + * Formats a numeric value for display in cards + */ + _formatDisplayValue(value) { + if (value === null || value === undefined || isNaN(value)) return '--'; + if (Math.abs(value) >= 1000) { + return value.toFixed(0); + } else if (Math.abs(value) >= 100) { + return value.toFixed(1); + } else { + return value.toFixed(2); } } @@ -181,8 +209,6 @@ class Indicator { } updateLine(name, data, value_name) { - console.log('indicators[68]: updateLine takes:(name, data, value_name)'); - console.log(name, data, value_name); // Check if the data is a multi-value object if (typeof data === 'object' && data !== null && value_name in data) { @@ -263,7 +289,6 @@ class SMA extends Indicator { init(data) { this.setLine('line', data, 'value'); - console.log('line data', data); } update(data) {