Improve backtest time handling and indicator warmup
- Add UTC time formatter to charts for consistent time display - Show both local and UTC times in backtest trades table - Add indicator warmup period calculation to fetch extra candles - Append 'Z' suffix to trade/alert timestamps to indicate UTC - Add get_available_balance to strategy execution context Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
307f251576
commit
24fcb56c38
|
|
@ -90,6 +90,7 @@ class StrategyInstance:
|
|||
'set_exit': self.set_exit,
|
||||
'set_available_strategy_balance': self.set_available_strategy_balance,
|
||||
'get_current_balance': self.get_current_balance,
|
||||
'get_available_balance': self.get_available_balance,
|
||||
'get_available_strategy_balance': self.get_available_strategy_balance,
|
||||
'starting_balance': self.starting_balance,
|
||||
'current_balance': self.current_balance,
|
||||
|
|
@ -373,7 +374,13 @@ class StrategyInstance:
|
|||
|
||||
# Call the 'next()' method if defined
|
||||
if 'next' in self.exec_context and callable(self.exec_context['next']):
|
||||
# Log flags before execution for debugging
|
||||
logger.debug(f"[STRATEGY EXEC] Flags before next(): {self.flags}")
|
||||
logger.debug(f"[STRATEGY EXEC] Variables before next(): {self.variables}")
|
||||
self.exec_context['next']()
|
||||
# Log flags after execution for debugging
|
||||
logger.debug(f"[STRATEGY EXEC] Flags after next(): {self.flags}")
|
||||
logger.debug(f"[STRATEGY EXEC] Variables after next(): {self.variables}")
|
||||
else:
|
||||
logger.error(
|
||||
f"'next' method not defined in generated_code for StrategyInstance '{self.strategy_instance_id}'."
|
||||
|
|
|
|||
|
|
@ -168,10 +168,19 @@ class BacktestStrategyInstance(StrategyInstance):
|
|||
|
||||
# Retrieve the value at the current index
|
||||
value = df.iloc[idx].get(output_field)
|
||||
indicator_time = df.iloc[idx].get('time', 'N/A')
|
||||
|
||||
# Get current candle time for comparison
|
||||
candle_time = None
|
||||
if self.backtrader_strategy and self.backtrader_strategy.data:
|
||||
try:
|
||||
candle_time = self.backtrader_strategy.data.datetime.datetime(0)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Log indicator values for debugging (first 10 and every 50th)
|
||||
if idx < 10 or idx % 50 == 0:
|
||||
logger.info(f"[BACKTEST] process_indicator('{indicator_name}', '{output_field}') at idx={idx}: raw_value={value}")
|
||||
logger.info(f"[BACKTEST] process_indicator('{indicator_name}', '{output_field}') at idx={idx}: value={value}, indicator_time={indicator_time}, candle_time={candle_time}")
|
||||
|
||||
if pd.isna(value):
|
||||
# Check if we have a cached last valid value
|
||||
|
|
@ -320,7 +329,8 @@ class BacktestStrategyInstance(StrategyInstance):
|
|||
"""
|
||||
timestamp = self.get_current_candle_datetime()
|
||||
alert = {
|
||||
'timestamp': timestamp.isoformat() if timestamp else None,
|
||||
# Append 'Z' to indicate UTC timezone (Backtrader uses UTC internally)
|
||||
'timestamp': (timestamp.isoformat() + 'Z') if timestamp else None,
|
||||
'message': message
|
||||
}
|
||||
self.collected_alerts.append(alert)
|
||||
|
|
|
|||
|
|
@ -442,6 +442,62 @@ class Backtester:
|
|||
|
||||
return precomputed_indicators
|
||||
|
||||
def _calculate_warmup_period(self, indicators_definitions: list, user_name: str) -> int:
|
||||
"""
|
||||
Calculate the maximum warmup period needed based on indicator periods.
|
||||
|
||||
:param indicators_definitions: List of indicator definitions from strategy
|
||||
:param user_name: Username for looking up indicator configs
|
||||
:return: Maximum warmup period in candles
|
||||
"""
|
||||
import json as json_module
|
||||
|
||||
max_period = 0
|
||||
|
||||
user_id = self.data_cache.get_datacache_item(
|
||||
item_name='id',
|
||||
cache_name='users',
|
||||
filter_vals=('user_name', user_name)
|
||||
)
|
||||
|
||||
for indicator_def in indicators_definitions:
|
||||
indicator_name = indicator_def.get('name')
|
||||
if not indicator_name:
|
||||
continue
|
||||
|
||||
try:
|
||||
indicators = self.data_cache.get_rows_from_datacache(
|
||||
cache_name='indicators',
|
||||
filter_vals=[('creator', str(user_id)), ('name', indicator_name)]
|
||||
)
|
||||
|
||||
if indicators.empty:
|
||||
continue
|
||||
|
||||
indicator = indicators.iloc[0]
|
||||
properties = json_module.loads(indicator['properties']) if isinstance(indicator['properties'], str) else indicator['properties']
|
||||
|
||||
# Get period from properties (most indicators use 'period')
|
||||
period = properties.get('period', 0)
|
||||
if period > max_period:
|
||||
max_period = period
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not get period for indicator '{indicator_name}': {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"[BACKTEST] Maximum indicator warmup period: {max_period}")
|
||||
return max_period
|
||||
|
||||
def _get_timeframe_minutes(self, timeframe: str) -> int:
|
||||
"""Convert timeframe string to minutes."""
|
||||
timeframe_map = {
|
||||
'1m': 1, '3m': 3, '5m': 5, '15m': 15, '30m': 30,
|
||||
'1h': 60, '2h': 120, '4h': 240, '6h': 360, '12h': 720,
|
||||
'1d': 1440, '1w': 10080
|
||||
}
|
||||
return timeframe_map.get(timeframe.lower(), 60) # Default to 1h
|
||||
|
||||
def prepare_backtest_data(self, msg_data: dict, strategy_components: dict) -> tuple:
|
||||
"""
|
||||
Prepare the data feed and precomputed indicators for backtesting.
|
||||
|
|
@ -466,30 +522,70 @@ class Backtester:
|
|||
main_source = data_sources[0]
|
||||
logger.info(f"Using main_source for backtest: {main_source}")
|
||||
|
||||
# Prepare the main data feed
|
||||
start_date = msg_data.get('start_date', '2023-01-01T00:00')
|
||||
logger.info(f"Backtest start_date: {start_date}")
|
||||
data_feed = self.prepare_data_feed(start_date, main_source, user_name)
|
||||
# Calculate warmup period needed for indicators
|
||||
indicators_definitions = strategy_components.get('indicators', [])
|
||||
warmup_candles = self._calculate_warmup_period(indicators_definitions, user_name)
|
||||
|
||||
# Get timeframe to calculate how far back to fetch for warmup
|
||||
timeframe = main_source.get('timeframe', '1h')
|
||||
timeframe_minutes = self._get_timeframe_minutes(timeframe)
|
||||
warmup_minutes = warmup_candles * timeframe_minutes
|
||||
|
||||
# Prepare the main data feed with extended start for warmup
|
||||
original_start_date = msg_data.get('start_date', '2023-01-01T00:00')
|
||||
logger.info(f"Backtest original start_date: {original_start_date}")
|
||||
|
||||
# Calculate adjusted start date for indicator warmup
|
||||
original_start_dt = dt.datetime.strptime(original_start_date, '%Y-%m-%dT%H:%M')
|
||||
adjusted_start_dt = original_start_dt - dt.timedelta(minutes=warmup_minutes)
|
||||
adjusted_start_date = adjusted_start_dt.strftime('%Y-%m-%dT%H:%M')
|
||||
|
||||
logger.info(f"[BACKTEST] Fetching data from {adjusted_start_date} (adjusted for {warmup_candles} warmup candles)")
|
||||
|
||||
data_feed = self.prepare_data_feed(adjusted_start_date, main_source, user_name)
|
||||
|
||||
if data_feed.empty:
|
||||
logger.error("Data feed could not be prepared. Please check the data source.")
|
||||
raise ValueError("Data feed could not be prepared. Please check the data source.")
|
||||
|
||||
# Precompute indicator values
|
||||
indicators_definitions = strategy_components.get('indicators', [])
|
||||
# Precompute indicator values on the full dataset (including warmup candles)
|
||||
precomputed_indicators = self.precompute_indicators(indicators_definitions, user_name, data_feed)
|
||||
|
||||
# Align data feed with indicator data by trimming to match the shortest indicator
|
||||
# Indicators skip their warmup period, so they have fewer rows than the raw candle data
|
||||
# Now trim BOTH the data feed AND indicators to start at the user's original start_date
|
||||
# This ensures the first indicator values in the backtest have full warmup context
|
||||
if precomputed_indicators:
|
||||
min_indicator_len = min(len(df) for df in precomputed_indicators.values())
|
||||
original_len = len(data_feed)
|
||||
# Find where the original start_date falls in the data
|
||||
original_start_unix = original_start_dt.replace(tzinfo=dt.timezone.utc).timestamp()
|
||||
|
||||
if min_indicator_len < original_len:
|
||||
# Trim the data feed from the beginning to align with indicators
|
||||
warmup_period = original_len - min_indicator_len
|
||||
data_feed = data_feed.iloc[warmup_period:].reset_index(drop=True)
|
||||
logger.info(f"[BACKTEST] Trimmed data feed from {original_len} to {len(data_feed)} rows to align with indicators (warmup period: {warmup_period})")
|
||||
# Find the index where we should start the backtest
|
||||
backtest_start_idx = 0
|
||||
for idx, row in data_feed.iterrows():
|
||||
if row['time'] >= original_start_unix:
|
||||
backtest_start_idx = idx
|
||||
break
|
||||
|
||||
# Calculate how many indicator rows to skip
|
||||
min_indicator_len = min(len(df) for df in precomputed_indicators.values())
|
||||
original_feed_len = len(data_feed)
|
||||
indicator_warmup = original_feed_len - min_indicator_len
|
||||
|
||||
# The effective backtest start is max of (user's start, indicator warmup)
|
||||
effective_start_idx = max(backtest_start_idx, indicator_warmup)
|
||||
|
||||
logger.info(f"[BACKTEST] Original data length: {original_feed_len}, indicator warmup: {indicator_warmup}, user start idx: {backtest_start_idx}, effective start: {effective_start_idx}")
|
||||
|
||||
# Trim data feed
|
||||
if effective_start_idx > 0:
|
||||
data_feed = data_feed.iloc[effective_start_idx:].reset_index(drop=True)
|
||||
logger.info(f"[BACKTEST] Trimmed data feed to {len(data_feed)} rows starting from effective start")
|
||||
|
||||
# Trim indicators to match
|
||||
# Indicators already have warmup_period fewer rows, so we need to adjust
|
||||
indicator_trim = effective_start_idx - indicator_warmup
|
||||
if indicator_trim > 0:
|
||||
for name, df in precomputed_indicators.items():
|
||||
precomputed_indicators[name] = df.iloc[indicator_trim:].reset_index(drop=True)
|
||||
logger.info(f"[BACKTEST] Trimmed indicator '{name}' to {len(precomputed_indicators[name])} rows")
|
||||
|
||||
logger.info("Backtest data prepared successfully.")
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,8 @@ class MappedStrategy(bt.Strategy):
|
|||
if trade.isopen:
|
||||
# Trade just opened - use current bar's datetime from data feed
|
||||
current_dt = self.data.datetime.datetime(0)
|
||||
open_datetime = current_dt.isoformat() if current_dt else None
|
||||
# Append 'Z' to indicate UTC timezone (Backtrader uses UTC internally)
|
||||
open_datetime = (current_dt.isoformat() + 'Z') if current_dt else None
|
||||
# Debug logging
|
||||
raw_dt = self.data.datetime[0]
|
||||
logger.info(f"[DEBUG] Trade open - raw datetime[0]={raw_dt}, converted={current_dt}, iso={open_datetime}")
|
||||
|
|
@ -105,7 +106,8 @@ class MappedStrategy(bt.Strategy):
|
|||
elif trade.isclosed:
|
||||
# Trade just closed - use current bar's datetime from data feed
|
||||
current_dt = self.data.datetime.datetime(0)
|
||||
close_datetime = current_dt.isoformat() if current_dt else None
|
||||
# Append 'Z' to indicate UTC timezone (Backtrader uses UTC internally)
|
||||
close_datetime = (current_dt.isoformat() + 'Z') if current_dt else None
|
||||
self.log(f"TRADE CLOSED, GROSS P/L: {trade.pnl}, NET P/L: {trade.pnlcomm}")
|
||||
# Retrieve open trade details
|
||||
trade_info = self.open_trades.pop(trade.ref, {})
|
||||
|
|
|
|||
|
|
@ -686,17 +686,28 @@ class Backtesting {
|
|||
return `${year}-${month}-${day}T${hours}:${minutes}`;
|
||||
}
|
||||
|
||||
// Format trade datetime for display in trades table
|
||||
// Format trade datetime for display in trades table (shows both local and UTC)
|
||||
formatTradeDateTime(dateTimeStr) {
|
||||
if (!dateTimeStr) return 'N/A';
|
||||
try {
|
||||
const date = new Date(dateTimeStr);
|
||||
const pad = (num) => num.toString().padStart(2, '0');
|
||||
const month = pad(date.getMonth() + 1);
|
||||
const day = pad(date.getDate());
|
||||
const hours = pad(date.getHours());
|
||||
const minutes = pad(date.getMinutes());
|
||||
return `${month}/${day} ${hours}:${minutes}`;
|
||||
|
||||
// Local time
|
||||
const localMonth = pad(date.getMonth() + 1);
|
||||
const localDay = pad(date.getDate());
|
||||
const localHours = pad(date.getHours());
|
||||
const localMinutes = pad(date.getMinutes());
|
||||
const localStr = `${localMonth}/${localDay} ${localHours}:${localMinutes}`;
|
||||
|
||||
// UTC time
|
||||
const utcMonth = pad(date.getUTCMonth() + 1);
|
||||
const utcDay = pad(date.getUTCDate());
|
||||
const utcHours = pad(date.getUTCHours());
|
||||
const utcMinutes = pad(date.getUTCMinutes());
|
||||
const utcStr = `${utcMonth}/${utcDay} ${utcHours}:${utcMinutes}`;
|
||||
|
||||
return `${localStr} (UTC: ${utcStr})`;
|
||||
} catch (e) {
|
||||
return dateTimeStr;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ class Charts {
|
|||
crosshair: {
|
||||
mode: LightweightCharts.CrosshairMode.Normal,
|
||||
},
|
||||
priceScale: {
|
||||
rightPriceScale: {
|
||||
borderColor: 'rgba(197, 203, 206, 0.8)',
|
||||
},
|
||||
timeScale: {
|
||||
|
|
@ -89,8 +89,22 @@ class Charts {
|
|||
secondsVisible: false,
|
||||
barSpacing: 6
|
||||
},
|
||||
handleScroll: true
|
||||
handleScroll: true,
|
||||
localization: {
|
||||
// Display times in UTC to match server data
|
||||
timeFormatter: (timestamp) => {
|
||||
const date = new Date(timestamp * 1000);
|
||||
const day = date.getUTCDate().toString().padStart(2, '0');
|
||||
const months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
||||
const month = months[date.getUTCMonth()];
|
||||
const year = date.getUTCFullYear().toString().slice(-2);
|
||||
const hours = date.getUTCHours().toString().padStart(2, '0');
|
||||
const minutes = date.getUTCMinutes().toString().padStart(2, '0');
|
||||
return `${day}/${month}/${year} ${hours}:${minutes}`;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return chart;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue