From 24fcb56c38ac4558255d889615dd34de9ff4a860 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 8 Mar 2026 00:38:22 -0400 Subject: [PATCH] Improve backtest time handling and indicator warmup - Add UTC time formatter to charts for consistent time display - Show both local and UTC times in backtest trades table - Add indicator warmup period calculation to fetch extra candles - Append 'Z' suffix to trade/alert timestamps to indicate UTC - Add get_available_balance to strategy execution context Co-Authored-By: Claude Opus 4.5 --- src/StrategyInstance.py | 7 ++ src/backtest_strategy_instance.py | 14 +++- src/backtesting.py | 126 ++++++++++++++++++++++++++---- src/mapped_strategy.py | 6 +- src/static/backtesting.js | 23 ++++-- src/static/charts.js | 18 ++++- 6 files changed, 167 insertions(+), 27 deletions(-) diff --git a/src/StrategyInstance.py b/src/StrategyInstance.py index e2c6118..3a797db 100644 --- a/src/StrategyInstance.py +++ b/src/StrategyInstance.py @@ -90,6 +90,7 @@ class StrategyInstance: 'set_exit': self.set_exit, 'set_available_strategy_balance': self.set_available_strategy_balance, 'get_current_balance': self.get_current_balance, + 'get_available_balance': self.get_available_balance, 'get_available_strategy_balance': self.get_available_strategy_balance, 'starting_balance': self.starting_balance, 'current_balance': self.current_balance, @@ -373,7 +374,13 @@ class StrategyInstance: # Call the 'next()' method if defined if 'next' in self.exec_context and callable(self.exec_context['next']): + # Log flags before execution for debugging + logger.debug(f"[STRATEGY EXEC] Flags before next(): {self.flags}") + logger.debug(f"[STRATEGY EXEC] Variables before next(): {self.variables}") self.exec_context['next']() + # Log flags after execution for debugging + logger.debug(f"[STRATEGY EXEC] Flags after next(): {self.flags}") + logger.debug(f"[STRATEGY EXEC] Variables after next(): {self.variables}") else: logger.error( f"'next' method not defined in generated_code for StrategyInstance '{self.strategy_instance_id}'." diff --git a/src/backtest_strategy_instance.py b/src/backtest_strategy_instance.py index 764e880..39ae834 100644 --- a/src/backtest_strategy_instance.py +++ b/src/backtest_strategy_instance.py @@ -168,10 +168,19 @@ class BacktestStrategyInstance(StrategyInstance): # Retrieve the value at the current index value = df.iloc[idx].get(output_field) + indicator_time = df.iloc[idx].get('time', 'N/A') + + # Get current candle time for comparison + candle_time = None + if self.backtrader_strategy and self.backtrader_strategy.data: + try: + candle_time = self.backtrader_strategy.data.datetime.datetime(0) + except: + pass # Log indicator values for debugging (first 10 and every 50th) if idx < 10 or idx % 50 == 0: - logger.info(f"[BACKTEST] process_indicator('{indicator_name}', '{output_field}') at idx={idx}: raw_value={value}") + logger.info(f"[BACKTEST] process_indicator('{indicator_name}', '{output_field}') at idx={idx}: value={value}, indicator_time={indicator_time}, candle_time={candle_time}") if pd.isna(value): # Check if we have a cached last valid value @@ -320,7 +329,8 @@ class BacktestStrategyInstance(StrategyInstance): """ timestamp = self.get_current_candle_datetime() alert = { - 'timestamp': timestamp.isoformat() if timestamp else None, + # Append 'Z' to indicate UTC timezone (Backtrader uses UTC internally) + 'timestamp': (timestamp.isoformat() + 'Z') if timestamp else None, 'message': message } self.collected_alerts.append(alert) diff --git a/src/backtesting.py b/src/backtesting.py index 4a03662..e348f50 100644 --- a/src/backtesting.py +++ b/src/backtesting.py @@ -442,6 +442,62 @@ class Backtester: return precomputed_indicators + def _calculate_warmup_period(self, indicators_definitions: list, user_name: str) -> int: + """ + Calculate the maximum warmup period needed based on indicator periods. + + :param indicators_definitions: List of indicator definitions from strategy + :param user_name: Username for looking up indicator configs + :return: Maximum warmup period in candles + """ + import json as json_module + + max_period = 0 + + user_id = self.data_cache.get_datacache_item( + item_name='id', + cache_name='users', + filter_vals=('user_name', user_name) + ) + + for indicator_def in indicators_definitions: + indicator_name = indicator_def.get('name') + if not indicator_name: + continue + + try: + indicators = self.data_cache.get_rows_from_datacache( + cache_name='indicators', + filter_vals=[('creator', str(user_id)), ('name', indicator_name)] + ) + + if indicators.empty: + continue + + indicator = indicators.iloc[0] + properties = json_module.loads(indicator['properties']) if isinstance(indicator['properties'], str) else indicator['properties'] + + # Get period from properties (most indicators use 'period') + period = properties.get('period', 0) + if period > max_period: + max_period = period + + except Exception as e: + logger.warning(f"Could not get period for indicator '{indicator_name}': {e}") + continue + + logger.info(f"[BACKTEST] Maximum indicator warmup period: {max_period}") + return max_period + + def _get_timeframe_minutes(self, timeframe: str) -> int: + """Convert timeframe string to minutes.""" + timeframe_map = { + '1m': 1, '3m': 3, '5m': 5, '15m': 15, '30m': 30, + '1h': 60, '2h': 120, '4h': 240, '6h': 360, '12h': 720, + '1d': 1440, '1w': 10080 + } + return timeframe_map.get(timeframe.lower(), 60) # Default to 1h + def prepare_backtest_data(self, msg_data: dict, strategy_components: dict) -> tuple: """ Prepare the data feed and precomputed indicators for backtesting. @@ -466,30 +522,70 @@ class Backtester: main_source = data_sources[0] logger.info(f"Using main_source for backtest: {main_source}") - # Prepare the main data feed - start_date = msg_data.get('start_date', '2023-01-01T00:00') - logger.info(f"Backtest start_date: {start_date}") - data_feed = self.prepare_data_feed(start_date, main_source, user_name) + # Calculate warmup period needed for indicators + indicators_definitions = strategy_components.get('indicators', []) + warmup_candles = self._calculate_warmup_period(indicators_definitions, user_name) + + # Get timeframe to calculate how far back to fetch for warmup + timeframe = main_source.get('timeframe', '1h') + timeframe_minutes = self._get_timeframe_minutes(timeframe) + warmup_minutes = warmup_candles * timeframe_minutes + + # Prepare the main data feed with extended start for warmup + original_start_date = msg_data.get('start_date', '2023-01-01T00:00') + logger.info(f"Backtest original start_date: {original_start_date}") + + # Calculate adjusted start date for indicator warmup + original_start_dt = dt.datetime.strptime(original_start_date, '%Y-%m-%dT%H:%M') + adjusted_start_dt = original_start_dt - dt.timedelta(minutes=warmup_minutes) + adjusted_start_date = adjusted_start_dt.strftime('%Y-%m-%dT%H:%M') + + logger.info(f"[BACKTEST] Fetching data from {adjusted_start_date} (adjusted for {warmup_candles} warmup candles)") + + data_feed = self.prepare_data_feed(adjusted_start_date, main_source, user_name) if data_feed.empty: logger.error("Data feed could not be prepared. Please check the data source.") raise ValueError("Data feed could not be prepared. Please check the data source.") - # Precompute indicator values - indicators_definitions = strategy_components.get('indicators', []) + # Precompute indicator values on the full dataset (including warmup candles) precomputed_indicators = self.precompute_indicators(indicators_definitions, user_name, data_feed) - # Align data feed with indicator data by trimming to match the shortest indicator - # Indicators skip their warmup period, so they have fewer rows than the raw candle data + # Now trim BOTH the data feed AND indicators to start at the user's original start_date + # This ensures the first indicator values in the backtest have full warmup context if precomputed_indicators: - min_indicator_len = min(len(df) for df in precomputed_indicators.values()) - original_len = len(data_feed) + # Find where the original start_date falls in the data + original_start_unix = original_start_dt.replace(tzinfo=dt.timezone.utc).timestamp() - if min_indicator_len < original_len: - # Trim the data feed from the beginning to align with indicators - warmup_period = original_len - min_indicator_len - data_feed = data_feed.iloc[warmup_period:].reset_index(drop=True) - logger.info(f"[BACKTEST] Trimmed data feed from {original_len} to {len(data_feed)} rows to align with indicators (warmup period: {warmup_period})") + # Find the index where we should start the backtest + backtest_start_idx = 0 + for idx, row in data_feed.iterrows(): + if row['time'] >= original_start_unix: + backtest_start_idx = idx + break + + # Calculate how many indicator rows to skip + min_indicator_len = min(len(df) for df in precomputed_indicators.values()) + original_feed_len = len(data_feed) + indicator_warmup = original_feed_len - min_indicator_len + + # The effective backtest start is max of (user's start, indicator warmup) + effective_start_idx = max(backtest_start_idx, indicator_warmup) + + logger.info(f"[BACKTEST] Original data length: {original_feed_len}, indicator warmup: {indicator_warmup}, user start idx: {backtest_start_idx}, effective start: {effective_start_idx}") + + # Trim data feed + if effective_start_idx > 0: + data_feed = data_feed.iloc[effective_start_idx:].reset_index(drop=True) + logger.info(f"[BACKTEST] Trimmed data feed to {len(data_feed)} rows starting from effective start") + + # Trim indicators to match + # Indicators already have warmup_period fewer rows, so we need to adjust + indicator_trim = effective_start_idx - indicator_warmup + if indicator_trim > 0: + for name, df in precomputed_indicators.items(): + precomputed_indicators[name] = df.iloc[indicator_trim:].reset_index(drop=True) + logger.info(f"[BACKTEST] Trimmed indicator '{name}' to {len(precomputed_indicators[name])} rows") logger.info("Backtest data prepared successfully.") diff --git a/src/mapped_strategy.py b/src/mapped_strategy.py index 382d9b4..99dbf0c 100644 --- a/src/mapped_strategy.py +++ b/src/mapped_strategy.py @@ -89,7 +89,8 @@ class MappedStrategy(bt.Strategy): if trade.isopen: # Trade just opened - use current bar's datetime from data feed current_dt = self.data.datetime.datetime(0) - open_datetime = current_dt.isoformat() if current_dt else None + # Append 'Z' to indicate UTC timezone (Backtrader uses UTC internally) + open_datetime = (current_dt.isoformat() + 'Z') if current_dt else None # Debug logging raw_dt = self.data.datetime[0] logger.info(f"[DEBUG] Trade open - raw datetime[0]={raw_dt}, converted={current_dt}, iso={open_datetime}") @@ -105,7 +106,8 @@ class MappedStrategy(bt.Strategy): elif trade.isclosed: # Trade just closed - use current bar's datetime from data feed current_dt = self.data.datetime.datetime(0) - close_datetime = current_dt.isoformat() if current_dt else None + # Append 'Z' to indicate UTC timezone (Backtrader uses UTC internally) + close_datetime = (current_dt.isoformat() + 'Z') if current_dt else None self.log(f"TRADE CLOSED, GROSS P/L: {trade.pnl}, NET P/L: {trade.pnlcomm}") # Retrieve open trade details trade_info = self.open_trades.pop(trade.ref, {}) diff --git a/src/static/backtesting.js b/src/static/backtesting.js index ffef3da..02d85df 100644 --- a/src/static/backtesting.js +++ b/src/static/backtesting.js @@ -686,17 +686,28 @@ class Backtesting { return `${year}-${month}-${day}T${hours}:${minutes}`; } - // Format trade datetime for display in trades table + // Format trade datetime for display in trades table (shows both local and UTC) formatTradeDateTime(dateTimeStr) { if (!dateTimeStr) return 'N/A'; try { const date = new Date(dateTimeStr); const pad = (num) => num.toString().padStart(2, '0'); - const month = pad(date.getMonth() + 1); - const day = pad(date.getDate()); - const hours = pad(date.getHours()); - const minutes = pad(date.getMinutes()); - return `${month}/${day} ${hours}:${minutes}`; + + // Local time + const localMonth = pad(date.getMonth() + 1); + const localDay = pad(date.getDate()); + const localHours = pad(date.getHours()); + const localMinutes = pad(date.getMinutes()); + const localStr = `${localMonth}/${localDay} ${localHours}:${localMinutes}`; + + // UTC time + const utcMonth = pad(date.getUTCMonth() + 1); + const utcDay = pad(date.getUTCDate()); + const utcHours = pad(date.getUTCHours()); + const utcMinutes = pad(date.getUTCMinutes()); + const utcStr = `${utcMonth}/${utcDay} ${utcHours}:${utcMinutes}`; + + return `${localStr} (UTC: ${utcStr})`; } catch (e) { return dateTimeStr; } diff --git a/src/static/charts.js b/src/static/charts.js index e4f1348..5eff568 100644 --- a/src/static/charts.js +++ b/src/static/charts.js @@ -80,7 +80,7 @@ class Charts { crosshair: { mode: LightweightCharts.CrosshairMode.Normal, }, - priceScale: { + rightPriceScale: { borderColor: 'rgba(197, 203, 206, 0.8)', }, timeScale: { @@ -89,8 +89,22 @@ class Charts { secondsVisible: false, barSpacing: 6 }, - handleScroll: true + handleScroll: true, + localization: { + // Display times in UTC to match server data + timeFormatter: (timestamp) => { + const date = new Date(timestamp * 1000); + const day = date.getUTCDate().toString().padStart(2, '0'); + const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']; + const month = months[date.getUTCMonth()]; + const year = date.getUTCFullYear().toString().slice(-2); + const hours = date.getUTCHours().toString().padStart(2, '0'); + const minutes = date.getUTCMinutes().toString().padStart(2, '0'); + return `${day}/${month}/${year} ${hours}:${minutes}`; + } + } }); + return chart; }