Fix stale data served for limit-only requests

Add freshness check to _detect_gaps() in all cache layers (memory,
database, async_database). For limit-only requests (no start/end),
verify the most recent candle is within 2 intervals of current time.
If stale, flag as a gap so fresh data is fetched from exchange.

This fixes the issue where EDM served hours-old cached data instead
of fetching current candles when clients requested "latest N candles".

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/exchange_data_manager/cache/async_database.py

@@ -205,6 +205,7 @@ class AsyncDatabaseCache:
             return []
 
         from ..utils.timeframes import timeframe_to_seconds
+        import time as time_module
 
         try:
             interval = timeframe_to_seconds(timeframe)
@@ -229,6 +230,17 @@ class AsyncDatabaseCache:
         if end is not None and sorted_candles[-1].time + interval <= end:
             gaps.append((sorted_candles[-1].time + interval, end))
 
+        # FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current
+        # If the most recent candle is too old, flag it as a gap so fresh data is fetched
+        if start is None and end is None:
+            now = int(time_module.time())
+            most_recent = sorted_candles[-1].time
+            # Consider data stale if most recent candle is older than 2 intervals
+            staleness_threshold = interval * 2
+            if now - most_recent > staleness_threshold:
+                # Data is stale - add gap from last candle to now
+                gaps.append((most_recent + interval, now))
+
         return gaps
 
     async def put(

src/exchange_data_manager/cache/database.py

@@ -173,6 +173,7 @@ class DatabaseCache:
             return []
 
         from ..candles.assembler import timeframe_to_seconds
+        import time as time_module
 
         try:
             interval = timeframe_to_seconds(timeframe)
@@ -197,6 +198,17 @@ class DatabaseCache:
         if end is not None and sorted_candles[-1].time + interval <= end:
             gaps.append((sorted_candles[-1].time + interval, end))
 
+        # FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current
+        # If the most recent candle is too old, flag it as a gap so fresh data is fetched
+        if start is None and end is None:
+            now = int(time_module.time())
+            most_recent = sorted_candles[-1].time
+            # Consider data stale if most recent candle is older than 2 intervals
+            staleness_threshold = interval * 2
+            if now - most_recent > staleness_threshold:
+                # Data is stale - add gap from last candle to now
+                gaps.append((most_recent + interval, now))
+
         return gaps
 
     def put(self, exchange: str, symbol: str, timeframe: str, candles: List[Candle]):

src/exchange_data_manager/cache/memory.py

@@ -189,6 +189,18 @@ class MemoryCache:
         if end is not None and candles[-1].time + interval <= end:
             gaps.append((candles[-1].time + interval, end))
 
+        # FRESHNESS CHECK: For limit-only requests (no start/end), verify data is current
+        # If the most recent candle is too old, flag it as a gap so fresh data is fetched
+        if start is None and end is None:
+            now = int(time.time())
+            most_recent = candles[-1].time
+            # Consider data stale if most recent candle is older than 2 intervals
+            # This ensures we fetch fresh data for "get latest N candles" requests
+            staleness_threshold = interval * 2
+            if now - most_recent > staleness_threshold:
+                # Data is stale - add gap from last candle to now
+                gaps.append((most_recent + interval, now))
+
         return gaps
 
     def put(self, cache_key: str, candles: List[Candle]):

tests/test_cache.py

@@ -494,3 +494,104 @@ class TestMemoryCacheBinarySearch:
         # Query after all candles
         result, _ = cache.get("test:TEST:1m", start=301, end=400)
         assert len(result) == 0
+
+
+class TestFreshnessCheck:
+    """Tests for stale data detection in limit-only requests."""
+
+    def test_memory_cache_detects_stale_data_limit_only(self):
+        """Test that memory cache detects stale data for limit-only requests."""
+        import time as time_module
+        cache = MemoryCache()
+
+        # Create candles that are 10 minutes old (stale for 1m timeframe)
+        now = int(time_module.time())
+        old_time = now - 600  # 10 minutes ago
+
+        candles = [
+            Candle(time=old_time - 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=old_time - 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+        ]
+        cache.put("binance:BTC/USDT:1m", candles)
+
+        # Limit-only request (no start/end) - should detect staleness
+        result, gaps = cache.get("binance:BTC/USDT:1m", limit=100)
+
+        assert len(result) == 3
+        # Should have a gap indicating data is stale and needs refresh
+        assert len(gaps) == 1
+        # Gap should be from after last candle to "now"
+        assert gaps[0][0] == old_time + 60  # Start after last candle
+        assert gaps[0][1] >= now - 5  # End should be close to now (within 5s tolerance)
+
+    def test_memory_cache_fresh_data_no_gaps_limit_only(self):
+        """Test that fresh data returns no gaps for limit-only requests."""
+        import time as time_module
+        cache = MemoryCache()
+
+        # Create candles that are current (within 2 intervals)
+        now = int(time_module.time())
+        # Align to 1m boundary
+        aligned_now = now - (now % 60)
+
+        candles = [
+            Candle(time=aligned_now - 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=aligned_now - 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=aligned_now, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+        ]
+        cache.put("binance:BTC/USDT:1m", candles)
+
+        # Limit-only request - fresh data should have no gaps
+        result, gaps = cache.get("binance:BTC/USDT:1m", limit=100)
+
+        assert len(result) == 3
+        assert len(gaps) == 0  # No staleness gap
+
+    def test_memory_cache_stale_5m_timeframe(self):
+        """Test staleness detection for 5m timeframe."""
+        import time as time_module
+        cache = MemoryCache()
+
+        # Create 5m candles that are 30 minutes old (stale: > 2 * 300s = 600s)
+        now = int(time_module.time())
+        old_time = now - 1800  # 30 minutes ago
+
+        candles = [
+            Candle(time=old_time - 600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=old_time - 300, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+        ]
+        cache.put("binance:BTC/USDT:5m", candles)
+
+        # Limit-only request - should detect staleness
+        result, gaps = cache.get("binance:BTC/USDT:5m", limit=100)
+
+        assert len(result) == 3
+        assert len(gaps) == 1  # Staleness gap detected
+
+    def test_range_request_not_affected_by_freshness_check(self):
+        """Test that range requests (with start/end) are not affected by freshness check."""
+        import time as time_module
+        cache = MemoryCache()
+
+        # Create old candles
+        old_time = 1709337600  # Fixed old timestamp
+
+        candles = [
+            Candle(time=old_time, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=old_time + 60, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+            Candle(time=old_time + 120, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
+        ]
+        cache.put("binance:BTC/USDT:1m", candles)
+
+        # Range request (with start/end) - freshness check should NOT apply
+        result, gaps = cache.get(
+            "binance:BTC/USDT:1m",
+            start=old_time,
+            end=old_time + 120
+        )
+
+        assert len(result) == 3
+        # No gaps because data covers the requested range completely
+        assert len(gaps) == 0

tests/test_cache_manager.py

@@ -1,7 +1,7 @@
 """Tests for CacheManager."""
 
 import pytest
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
 
 from exchange_data_manager.cache.manager import CacheManager
 from exchange_data_manager.config import DatabaseConfig
@@ -74,8 +74,12 @@ class TestCacheManagerColdCache:
         assert result == []
 
     @pytest.mark.asyncio
-    async def test_limit_only_caches_result(self, cache_manager):
+    @patch("exchange_data_manager.cache.memory.time.time")
+    async def test_limit_only_caches_result(self, mock_time, cache_manager):
         """Test that limit-only results are cached in memory."""
+        # Mock time to be close to test candle timestamps (prevents freshness check)
+        mock_time.return_value = 1709337660  # Just after the candle
+
         mock_connector = MockConnector()
         expected_candles = [
             Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
@@ -121,8 +125,12 @@ class TestCacheManagerSources:
     """Tests for source reporting and per-request connector overrides."""
 
     @pytest.mark.asyncio
-    async def test_get_candles_with_source_reports_memory(self, cache_manager):
+    @patch("exchange_data_manager.cache.memory.time.time")
+    async def test_get_candles_with_source_reports_memory(self, mock_time, cache_manager):
         """Memory-only responses should report memory source."""
+        # Mock time to be close to test candle timestamps (prevents freshness check)
+        mock_time.return_value = 1709337720  # Just after the second candle
+
         candles = [
             Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
             Candle(time=1709337660, open=50050.0, high=50200.0, low=50000.0, close=50150.0, volume=15.0),

tests/test_integration.py

@@ -84,8 +84,12 @@ class TestCacheManagerIntegration:
         mock_connector.fetch_candles.assert_called_once()
 
     @pytest.mark.asyncio
-    async def test_warm_cache_uses_memory(self, cache_manager):
+    @patch("exchange_data_manager.cache.memory.time.time")
+    async def test_warm_cache_uses_memory(self, mock_time, cache_manager):
         """Test that warm cache uses memory instead of exchange."""
+        # Mock time to be close to test candle timestamps (prevents freshness check)
+        mock_time.return_value = 1709337720  # Just after the second candle
+
         mock_connector = MockConnector()
         mock_connector.fetch_candles.return_value = [
             make_candle(1709337600),