Fix stale data served for limit-only requests
Add freshness check to _detect_gaps() in all cache layers (memory, database, async_database). For limit-only requests (no start/end), verify the most recent candle is within 2 intervals of current time. If stale, flag as a gap so fresh data is fetched from exchange. This fixes the issue where EDM served hours-old cached data instead of fetching current candles when clients requested "latest N candles". Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
d2cd47ea95
commit
958168b3c9
|
|
@ -205,6 +205,7 @@ class AsyncDatabaseCache:
|
|||
return []
|
||||
|
||||
from ..utils.timeframes import timeframe_to_seconds
|
||||
import time as time_module
|
||||
|
||||
try:
|
||||
interval = timeframe_to_seconds(timeframe)
|
||||
|
|
@ -229,6 +230,17 @@ class AsyncDatabaseCache:
|
|||
if end is not None and sorted_candles[-1].time + interval <= end:
|
||||
gaps.append((sorted_candles[-1].time + interval, end))
|
||||
|
||||
# FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current
|
||||
# If the most recent candle is too old, flag it as a gap so fresh data is fetched
|
||||
if start is None and end is None:
|
||||
now = int(time_module.time())
|
||||
most_recent = sorted_candles[-1].time
|
||||
# Consider data stale if most recent candle is older than 2 intervals
|
||||
staleness_threshold = interval * 2
|
||||
if now - most_recent > staleness_threshold:
|
||||
# Data is stale - add gap from last candle to now
|
||||
gaps.append((most_recent + interval, now))
|
||||
|
||||
return gaps
|
||||
|
||||
async def put(
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ class DatabaseCache:
|
|||
return []
|
||||
|
||||
from ..candles.assembler import timeframe_to_seconds
|
||||
import time as time_module
|
||||
|
||||
try:
|
||||
interval = timeframe_to_seconds(timeframe)
|
||||
|
|
@ -197,6 +198,17 @@ class DatabaseCache:
|
|||
if end is not None and sorted_candles[-1].time + interval <= end:
|
||||
gaps.append((sorted_candles[-1].time + interval, end))
|
||||
|
||||
# FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current
|
||||
# If the most recent candle is too old, flag it as a gap so fresh data is fetched
|
||||
if start is None and end is None:
|
||||
now = int(time_module.time())
|
||||
most_recent = sorted_candles[-1].time
|
||||
# Consider data stale if most recent candle is older than 2 intervals
|
||||
staleness_threshold = interval * 2
|
||||
if now - most_recent > staleness_threshold:
|
||||
# Data is stale - add gap from last candle to now
|
||||
gaps.append((most_recent + interval, now))
|
||||
|
||||
return gaps
|
||||
|
||||
def put(self, exchange: str, symbol: str, timeframe: str, candles: List[Candle]):
|
||||
|
|
|
|||
|
|
@ -189,6 +189,18 @@ class MemoryCache:
|
|||
if end is not None and candles[-1].time + interval <= end:
|
||||
gaps.append((candles[-1].time + interval, end))
|
||||
|
||||
# FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current
|
||||
# If the most recent candle is too old, flag it as a gap so fresh data is fetched
|
||||
if start is None and end is None:
|
||||
now = int(time.time())
|
||||
most_recent = candles[-1].time
|
||||
# Consider data stale if most recent candle is older than 2 intervals
|
||||
# This ensures we fetch fresh data for "get latest N candles" requests
|
||||
staleness_threshold = interval * 2
|
||||
if now - most_recent > staleness_threshold:
|
||||
# Data is stale - add gap from last candle to now
|
||||
gaps.append((most_recent + interval, now))
|
||||
|
||||
return gaps
|
||||
|
||||
def put(self, cache_key: str, candles: List[Candle]):
|
||||
|
|
|
|||
|
|
@ -494,3 +494,104 @@ class TestMemoryCacheBinarySearch:
|
|||
# Query after all candles
|
||||
result, _ = cache.get("test:TEST:1m", start=301, end=400)
|
||||
assert len(result) == 0
|
||||
|
||||
|
||||
class TestFreshnessCheck:
|
||||
"""Tests for stale data detection in limit-only requests."""
|
||||
|
||||
def test_memory_cache_detects_stale_data_limit_only(self):
|
||||
"""Test that memory cache detects stale data for limit-only requests."""
|
||||
import time as time_module
|
||||
cache = MemoryCache()
|
||||
|
||||
# Create candles that are 10 minutes old (stale for 1m timeframe)
|
||||
now = int(time_module.time())
|
||||
old_time = now - 600 # 10 minutes ago
|
||||
|
||||
candles = [
|
||||
Candle(time=old_time - 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=old_time - 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
]
|
||||
cache.put("binance:BTC/USDT:1m", candles)
|
||||
|
||||
# Limit-only request (no start/end) - should detect staleness
|
||||
result, gaps = cache.get("binance:BTC/USDT:1m", limit=100)
|
||||
|
||||
assert len(result) == 3
|
||||
# Should have a gap indicating data is stale and needs refresh
|
||||
assert len(gaps) == 1
|
||||
# Gap should be from after last candle to "now"
|
||||
assert gaps[0][0] == old_time + 60 # Start after last candle
|
||||
assert gaps[0][1] >= now - 5 # End should be close to now (within 5s tolerance)
|
||||
|
||||
def test_memory_cache_fresh_data_no_gaps_limit_only(self):
|
||||
"""Test that fresh data returns no gaps for limit-only requests."""
|
||||
import time as time_module
|
||||
cache = MemoryCache()
|
||||
|
||||
# Create candles that are current (within 2 intervals)
|
||||
now = int(time_module.time())
|
||||
# Align to 1m boundary
|
||||
aligned_now = now - (now % 60)
|
||||
|
||||
candles = [
|
||||
Candle(time=aligned_now - 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=aligned_now - 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=aligned_now, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
]
|
||||
cache.put("binance:BTC/USDT:1m", candles)
|
||||
|
||||
# Limit-only request - fresh data should have no gaps
|
||||
result, gaps = cache.get("binance:BTC/USDT:1m", limit=100)
|
||||
|
||||
assert len(result) == 3
|
||||
assert len(gaps) == 0 # No staleness gap
|
||||
|
||||
def test_memory_cache_stale_5m_timeframe(self):
|
||||
"""Test staleness detection for 5m timeframe."""
|
||||
import time as time_module
|
||||
cache = MemoryCache()
|
||||
|
||||
# Create 5m candles that are 30 minutes old (stale: > 2 * 300s = 600s)
|
||||
now = int(time_module.time())
|
||||
old_time = now - 1800 # 30 minutes ago
|
||||
|
||||
candles = [
|
||||
Candle(time=old_time - 600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=old_time - 300, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
]
|
||||
cache.put("binance:BTC/USDT:5m", candles)
|
||||
|
||||
# Limit-only request - should detect staleness
|
||||
result, gaps = cache.get("binance:BTC/USDT:5m", limit=100)
|
||||
|
||||
assert len(result) == 3
|
||||
assert len(gaps) == 1 # Staleness gap detected
|
||||
|
||||
def test_range_request_not_affected_by_freshness_check(self):
|
||||
"""Test that range requests (with start/end) are not affected by freshness check."""
|
||||
import time as time_module
|
||||
cache = MemoryCache()
|
||||
|
||||
# Create old candles
|
||||
old_time = 1709337600 # Fixed old timestamp
|
||||
|
||||
candles = [
|
||||
Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=old_time + 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
Candle(time=old_time + 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
||||
]
|
||||
cache.put("binance:BTC/USDT:1m", candles)
|
||||
|
||||
# Range request (with start/end) - freshness check should NOT apply
|
||||
result, gaps = cache.get(
|
||||
"binance:BTC/USDT:1m",
|
||||
start=old_time,
|
||||
end=old_time + 120
|
||||
)
|
||||
|
||||
assert len(result) == 3
|
||||
# No gaps because data covers the requested range completely
|
||||
assert len(gaps) == 0
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Tests for CacheManager."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from exchange_data_manager.cache.manager import CacheManager
|
||||
from exchange_data_manager.config import DatabaseConfig
|
||||
|
|
@ -74,8 +74,12 @@ class TestCacheManagerColdCache:
|
|||
assert result == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit_only_caches_result(self, cache_manager):
|
||||
@patch("exchange_data_manager.cache.memory.time.time")
|
||||
async def test_limit_only_caches_result(self, mock_time, cache_manager):
|
||||
"""Test that limit-only results are cached in memory."""
|
||||
# Mock time to be close to test candle timestamps (prevents freshness check)
|
||||
mock_time.return_value = 1709337660 # Just after the candle
|
||||
|
||||
mock_connector = MockConnector()
|
||||
expected_candles = [
|
||||
Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
|
||||
|
|
@ -121,8 +125,12 @@ class TestCacheManagerSources:
|
|||
"""Tests for source reporting and per-request connector overrides."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_candles_with_source_reports_memory(self, cache_manager):
|
||||
@patch("exchange_data_manager.cache.memory.time.time")
|
||||
async def test_get_candles_with_source_reports_memory(self, mock_time, cache_manager):
|
||||
"""Memory-only responses should report memory source."""
|
||||
# Mock time to be close to test candle timestamps (prevents freshness check)
|
||||
mock_time.return_value = 1709337720 # Just after the second candle
|
||||
|
||||
candles = [
|
||||
Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
|
||||
Candle(time=1709337660, open=50050.0, high=50200.0, low=50000.0, close=50150.0, volume=15.0),
|
||||
|
|
|
|||
|
|
@ -84,8 +84,12 @@ class TestCacheManagerIntegration:
|
|||
mock_connector.fetch_candles.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_warm_cache_uses_memory(self, cache_manager):
|
||||
@patch("exchange_data_manager.cache.memory.time.time")
|
||||
async def test_warm_cache_uses_memory(self, mock_time, cache_manager):
|
||||
"""Test that warm cache uses memory instead of exchange."""
|
||||
# Mock time to be close to test candle timestamps (prevents freshness check)
|
||||
mock_time.return_value = 1709337720 # Just after the second candle
|
||||
|
||||
mock_connector = MockConnector()
|
||||
mock_connector.fetch_candles.return_value = [
|
||||
make_candle(1709337600),
|
||||
|
|
|
|||
Loading…
Reference in New Issue