diff --git a/src/exchange_data_manager/cache/async_database.py b/src/exchange_data_manager/cache/async_database.py index f2785fa..d3bb38a 100644 --- a/src/exchange_data_manager/cache/async_database.py +++ b/src/exchange_data_manager/cache/async_database.py @@ -205,6 +205,7 @@ class AsyncDatabaseCache: return [] from ..utils.timeframes import timeframe_to_seconds + import time as time_module try: interval = timeframe_to_seconds(timeframe) @@ -229,6 +230,17 @@ class AsyncDatabaseCache: if end is not None and sorted_candles[-1].time + interval <= end: gaps.append((sorted_candles[-1].time + interval, end)) + # FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current + # If the most recent candle is too old, flag it as a gap so fresh data is fetched + if start is None and end is None: + now = int(time_module.time()) + most_recent = sorted_candles[-1].time + # Consider data stale if most recent candle is older than 2 intervals + staleness_threshold = interval * 2 + if now - most_recent > staleness_threshold: + # Data is stale - add gap from last candle to now + gaps.append((most_recent + interval, now)) + return gaps async def put( diff --git a/src/exchange_data_manager/cache/database.py b/src/exchange_data_manager/cache/database.py index 8a52bb3..98529ca 100644 --- a/src/exchange_data_manager/cache/database.py +++ b/src/exchange_data_manager/cache/database.py @@ -173,6 +173,7 @@ class DatabaseCache: return [] from ..candles.assembler import timeframe_to_seconds + import time as time_module try: interval = timeframe_to_seconds(timeframe) @@ -197,6 +198,17 @@ class DatabaseCache: if end is not None and sorted_candles[-1].time + interval <= end: gaps.append((sorted_candles[-1].time + interval, end)) + # FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current + # If the most recent candle is too old, flag it as a gap so fresh data is fetched + if start is None and end is None: + now = int(time_module.time()) + most_recent = sorted_candles[-1].time + # Consider data stale if most recent candle is older than 2 intervals + staleness_threshold = interval * 2 + if now - most_recent > staleness_threshold: + # Data is stale - add gap from last candle to now + gaps.append((most_recent + interval, now)) + return gaps def put(self, exchange: str, symbol: str, timeframe: str, candles: List[Candle]): diff --git a/src/exchange_data_manager/cache/memory.py b/src/exchange_data_manager/cache/memory.py index f4d3802..e850c56 100644 --- a/src/exchange_data_manager/cache/memory.py +++ b/src/exchange_data_manager/cache/memory.py @@ -189,6 +189,18 @@ class MemoryCache: if end is not None and candles[-1].time + interval <= end: gaps.append((candles[-1].time + interval, end)) + # FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current + # If the most recent candle is too old, flag it as a gap so fresh data is fetched + if start is None and end is None: + now = int(time.time()) + most_recent = candles[-1].time + # Consider data stale if most recent candle is older than 2 intervals + # This ensures we fetch fresh data for "get latest N candles" requests + staleness_threshold = interval * 2 + if now - most_recent > staleness_threshold: + # Data is stale - add gap from last candle to now + gaps.append((most_recent + interval, now)) + return gaps def put(self, cache_key: str, candles: List[Candle]): diff --git a/tests/test_cache.py b/tests/test_cache.py index c9a706f..31222d7 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -494,3 +494,104 @@ class TestMemoryCacheBinarySearch: # Query after all candles result, _ = cache.get("test:TEST:1m", start=301, end=400) assert len(result) == 0 + + +class TestFreshnessCheck: + """Tests for stale data detection in limit-only requests.""" + + def test_memory_cache_detects_stale_data_limit_only(self): + """Test that memory cache detects stale data for limit-only requests.""" + import time as time_module + cache = MemoryCache() + + # Create candles that are 10 minutes old (stale for 1m timeframe) + now = int(time_module.time()) + old_time = now - 600 # 10 minutes ago + + candles = [ + Candle(time=old_time - 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=old_time - 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + ] + cache.put("binance:BTC/USDT:1m", candles) + + # Limit-only request (no start/end) - should detect staleness + result, gaps = cache.get("binance:BTC/USDT:1m", limit=100) + + assert len(result) == 3 + # Should have a gap indicating data is stale and needs refresh + assert len(gaps) == 1 + # Gap should be from after last candle to "now" + assert gaps[0][0] == old_time + 60 # Start after last candle + assert gaps[0][1] >= now - 5 # End should be close to now (within 5s tolerance) + + def test_memory_cache_fresh_data_no_gaps_limit_only(self): + """Test that fresh data returns no gaps for limit-only requests.""" + import time as time_module + cache = MemoryCache() + + # Create candles that are current (within 2 intervals) + now = int(time_module.time()) + # Align to 1m boundary + aligned_now = now - (now % 60) + + candles = [ + Candle(time=aligned_now - 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=aligned_now - 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=aligned_now, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + ] + cache.put("binance:BTC/USDT:1m", candles) + + # Limit-only request - fresh data should have no gaps + result, gaps = cache.get("binance:BTC/USDT:1m", limit=100) + + assert len(result) == 3 + assert len(gaps) == 0 # No staleness gap + + def test_memory_cache_stale_5m_timeframe(self): + """Test staleness detection for 5m timeframe.""" + import time as time_module + cache = MemoryCache() + + # Create 5m candles that are 30 minutes old (stale: > 2 * 300s = 600s) + now = int(time_module.time()) + old_time = now - 1800 # 30 minutes ago + + candles = [ + Candle(time=old_time - 600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=old_time - 300, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + ] + cache.put("binance:BTC/USDT:5m", candles) + + # Limit-only request - should detect staleness + result, gaps = cache.get("binance:BTC/USDT:5m", limit=100) + + assert len(result) == 3 + assert len(gaps) == 1 # Staleness gap detected + + def test_range_request_not_affected_by_freshness_check(self): + """Test that range requests (with start/end) are not affected by freshness check.""" + import time as time_module + cache = MemoryCache() + + # Create old candles + old_time = 1709337600 # Fixed old timestamp + + candles = [ + Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=old_time + 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + Candle(time=old_time + 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), + ] + cache.put("binance:BTC/USDT:1m", candles) + + # Range request (with start/end) - freshness check should NOT apply + result, gaps = cache.get( + "binance:BTC/USDT:1m", + start=old_time, + end=old_time + 120 + ) + + assert len(result) == 3 + # No gaps because data covers the requested range completely + assert len(gaps) == 0 diff --git a/tests/test_cache_manager.py b/tests/test_cache_manager.py index 56deaf3..715fd25 100644 --- a/tests/test_cache_manager.py +++ b/tests/test_cache_manager.py @@ -1,7 +1,7 @@ """Tests for CacheManager.""" import pytest -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch from exchange_data_manager.cache.manager import CacheManager from exchange_data_manager.config import DatabaseConfig @@ -74,8 +74,12 @@ class TestCacheManagerColdCache: assert result == [] @pytest.mark.asyncio - async def test_limit_only_caches_result(self, cache_manager): + @patch("exchange_data_manager.cache.memory.time.time") + async def test_limit_only_caches_result(self, mock_time, cache_manager): """Test that limit-only results are cached in memory.""" + # Mock time to be close to test candle timestamps (prevents freshness check) + mock_time.return_value = 1709337660 # Just after the candle + mock_connector = MockConnector() expected_candles = [ Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0), @@ -121,8 +125,12 @@ class TestCacheManagerSources: """Tests for source reporting and per-request connector overrides.""" @pytest.mark.asyncio - async def test_get_candles_with_source_reports_memory(self, cache_manager): + @patch("exchange_data_manager.cache.memory.time.time") + async def test_get_candles_with_source_reports_memory(self, mock_time, cache_manager): """Memory-only responses should report memory source.""" + # Mock time to be close to test candle timestamps (prevents freshness check) + mock_time.return_value = 1709337720 # Just after the second candle + candles = [ Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0), Candle(time=1709337660, open=50050.0, high=50200.0, low=50000.0, close=50150.0, volume=15.0), diff --git a/tests/test_integration.py b/tests/test_integration.py index 8a6c675..cf798d7 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -84,8 +84,12 @@ class TestCacheManagerIntegration: mock_connector.fetch_candles.assert_called_once() @pytest.mark.asyncio - async def test_warm_cache_uses_memory(self, cache_manager): + @patch("exchange_data_manager.cache.memory.time.time") + async def test_warm_cache_uses_memory(self, mock_time, cache_manager): """Test that warm cache uses memory instead of exchange.""" + # Mock time to be close to test candle timestamps (prevents freshness check) + mock_time.return_value = 1709337720 # Just after the second candle + mock_connector = MockConnector() mock_connector.fetch_candles.return_value = [ make_candle(1709337600),