497 lines
18 KiB
Python
497 lines
18 KiB
Python
"""Tests for cache components."""
|
|
|
|
import pytest
|
|
import tempfile
|
|
import os
|
|
|
|
from exchange_data_manager.candles.models import Candle, CandleRequest
|
|
from exchange_data_manager.cache.memory import MemoryCache
|
|
from exchange_data_manager.cache.database import DatabaseCache
|
|
|
|
|
|
class TestMemoryCache:
|
|
"""Tests for MemoryCache class."""
|
|
|
|
def test_create_cache(self):
|
|
"""Test creating a memory cache."""
|
|
cache = MemoryCache(max_candles=100, ttl_seconds=60)
|
|
|
|
assert cache.max_candles == 100
|
|
assert cache.ttl_seconds == 60
|
|
|
|
def test_put_and_get_candles(self):
|
|
"""Test storing and retrieving candles."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
|
|
Candle(time=1709337660, open=50050.0, high=50200.0, low=50000.0, close=50150.0, volume=15.0),
|
|
Candle(time=1709337720, open=50150.0, high=50300.0, low=50100.0, close=50250.0, volume=12.0),
|
|
]
|
|
|
|
cache_key = "binance:BTC/USDT:1m"
|
|
cache.put(cache_key, candles)
|
|
|
|
result, gaps = cache.get(cache_key)
|
|
|
|
assert len(result) == 3
|
|
assert result[0].time == 1709337600
|
|
assert result[2].time == 1709337720
|
|
|
|
def test_get_nonexistent_returns_empty(self):
|
|
"""Test that getting non-existent data returns empty list."""
|
|
cache = MemoryCache()
|
|
|
|
result, gaps = cache.get("binance:BTC/USDT:1m")
|
|
|
|
assert result == []
|
|
|
|
def test_get_with_time_range(self):
|
|
"""Test getting candles within a time range."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
|
|
Candle(time=1709337660, open=50050.0, high=50200.0, low=50000.0, close=50150.0, volume=15.0),
|
|
Candle(time=1709337720, open=50150.0, high=50300.0, low=50100.0, close=50250.0, volume=12.0),
|
|
Candle(time=1709337780, open=50250.0, high=50400.0, low=50200.0, close=50350.0, volume=8.0),
|
|
]
|
|
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
# Get subset
|
|
result, gaps = cache.get(
|
|
"binance:BTC/USDT:1m",
|
|
start=1709337660,
|
|
end=1709337720,
|
|
)
|
|
|
|
assert len(result) == 2
|
|
assert result[0].time == 1709337660
|
|
assert result[1].time == 1709337720
|
|
|
|
def test_clear(self):
|
|
"""Test clearing the cache."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0)]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
cache.clear()
|
|
|
|
result, _ = cache.get("binance:BTC/USDT:1m")
|
|
assert result == []
|
|
|
|
def test_stats(self):
|
|
"""Test cache statistics."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
stats = cache.stats()
|
|
|
|
assert stats["num_entries"] == 1
|
|
assert stats["total_candles"] == 2
|
|
|
|
def test_update_candle(self):
|
|
"""Test updating a single candle."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
# Update the candle
|
|
updated = Candle(time=1709337600, open=50000.0, high=51000.0, low=49000.0, close=50500.0, volume=10.0)
|
|
cache.update_candle("binance:BTC/USDT:1m", updated)
|
|
|
|
result, _ = cache.get("binance:BTC/USDT:1m")
|
|
assert len(result) == 1
|
|
assert result[0].high == 51000.0
|
|
assert result[0].volume == 10.0
|
|
|
|
def test_merge_candles(self):
|
|
"""Test that putting candles merges with existing data."""
|
|
cache = MemoryCache()
|
|
|
|
# First batch
|
|
candles1 = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles1)
|
|
|
|
# Second batch with overlap and new
|
|
candles2 = [
|
|
Candle(time=1709337660, open=51000.0, high=51000.0, low=51000.0, close=51000.0, volume=2.0), # Update
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0), # New
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles2)
|
|
|
|
result, _ = cache.get("binance:BTC/USDT:1m")
|
|
assert len(result) == 3
|
|
# Check that second candle was updated
|
|
assert result[1].open == 51000.0
|
|
assert result[1].volume == 2.0
|
|
|
|
def test_single_candle_gap_detection(self):
|
|
"""Test that a single missing candle is detected as a gap."""
|
|
cache = MemoryCache()
|
|
|
|
# Candles with 1m interval but missing the second one
|
|
# time: 0, 60, 180 (missing 120)
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
# Missing: time=1709337660
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
result, gaps = cache.get(
|
|
"binance:BTC/USDT:1m",
|
|
start=1709337600,
|
|
end=1709337720,
|
|
)
|
|
|
|
assert len(result) == 2
|
|
# Should detect the gap for the single missing candle
|
|
assert len(gaps) == 1
|
|
assert gaps[0] == (1709337660, 1709337660) # Single candle gap
|
|
|
|
def test_single_candle_end_gap_detection(self):
|
|
"""Test that a single missing candle at the end is detected as a gap."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
result, gaps = cache.get(
|
|
"binance:BTC/USDT:1m",
|
|
start=1709337600,
|
|
end=1709337780,
|
|
)
|
|
|
|
assert len(result) == 3
|
|
assert len(gaps) == 1
|
|
assert gaps[0] == (1709337780, 1709337780) # Single candle end gap
|
|
|
|
|
|
class TestDatabaseCache:
|
|
"""Tests for DatabaseCache class."""
|
|
|
|
@pytest.fixture
|
|
def db_cache(self):
|
|
"""Create a temporary database for testing."""
|
|
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
|
|
db_path = f.name
|
|
|
|
cache = DatabaseCache(db_path=db_path)
|
|
|
|
yield cache
|
|
|
|
# Cleanup
|
|
if os.path.exists(db_path):
|
|
os.unlink(db_path)
|
|
|
|
def test_put_and_get_candles(self, db_cache):
|
|
"""Test storing and retrieving candles."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
|
|
Candle(time=1709337660, open=50050.0, high=50200.0, low=50000.0, close=50150.0, volume=15.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
|
|
result, gaps = db_cache.get("binance", "BTC/USDT", "1m")
|
|
|
|
assert len(result) == 2
|
|
assert result[0].time == 1709337600
|
|
assert result[1].time == 1709337660
|
|
|
|
def test_get_empty_returns_empty_list(self, db_cache):
|
|
"""Test that getting non-existent data returns empty list."""
|
|
result, gaps = db_cache.get("binance", "BTC/USDT", "1m")
|
|
|
|
assert result == []
|
|
|
|
def test_get_with_time_range(self, db_cache):
|
|
"""Test getting candles within a time range."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337780, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
|
|
result, gaps = db_cache.get(
|
|
"binance", "BTC/USDT", "1m",
|
|
start=1709337660,
|
|
end=1709337720,
|
|
)
|
|
|
|
assert len(result) == 2
|
|
assert result[0].time == 1709337660
|
|
assert result[1].time == 1709337720
|
|
|
|
def test_upsert_updates_existing(self, db_cache):
|
|
"""Test that storing duplicate timestamps updates existing records."""
|
|
candles1 = [
|
|
Candle(time=1709337600, open=50000.0, high=50100.0, low=49900.0, close=50050.0, volume=10.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles1)
|
|
|
|
# Store updated candle with same timestamp
|
|
candles2 = [
|
|
Candle(time=1709337600, open=51000.0, high=51100.0, low=50900.0, close=51050.0, volume=20.0),
|
|
]
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles2)
|
|
|
|
result, gaps = db_cache.get("binance", "BTC/USDT", "1m")
|
|
|
|
# Should only have one candle with updated values
|
|
assert len(result) == 1
|
|
assert result[0].open == 51000.0
|
|
assert result[0].volume == 20.0
|
|
|
|
def test_delete_all(self, db_cache):
|
|
"""Test deleting all data."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
db_cache.put("binance", "ETH/USDT", "1m", candles)
|
|
|
|
db_cache.delete()
|
|
|
|
result1, _ = db_cache.get("binance", "BTC/USDT", "1m")
|
|
result2, _ = db_cache.get("binance", "ETH/USDT", "1m")
|
|
|
|
assert result1 == []
|
|
assert result2 == []
|
|
|
|
def test_delete_filtered(self, db_cache):
|
|
"""Test deleting filtered data."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
db_cache.put("binance", "ETH/USDT", "1m", candles)
|
|
|
|
# Only delete BTC
|
|
db_cache.delete(exchange="binance", symbol="BTC/USDT")
|
|
|
|
result1, _ = db_cache.get("binance", "BTC/USDT", "1m")
|
|
result2, _ = db_cache.get("binance", "ETH/USDT", "1m")
|
|
|
|
assert result1 == []
|
|
assert len(result2) == 1
|
|
|
|
def test_stats(self, db_cache):
|
|
"""Test database statistics."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
|
|
stats = db_cache.stats()
|
|
|
|
assert stats["total_candles"] == 2
|
|
assert stats["num_entries"] == 1
|
|
|
|
def test_get_time_range(self, db_cache):
|
|
"""Test getting the time range of cached data."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
|
|
time_range = db_cache.get_time_range("binance", "BTC/USDT", "1m")
|
|
|
|
assert time_range is not None
|
|
assert time_range[0] == 1709337600 # min
|
|
assert time_range[1] == 1709337720 # max
|
|
|
|
def test_get_time_range_no_data(self, db_cache):
|
|
"""Test getting time range when no data exists."""
|
|
time_range = db_cache.get_time_range("binance", "BTC/USDT", "1m")
|
|
assert time_range is None
|
|
|
|
def test_single_candle_gap_detection(self, db_cache):
|
|
"""Test that a single missing candle is detected as a gap."""
|
|
# Candles with 1m interval but missing the second one
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
# Missing: time=1709337660
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
|
|
result, gaps = db_cache.get(
|
|
"binance", "BTC/USDT", "1m",
|
|
start=1709337600,
|
|
end=1709337720,
|
|
)
|
|
|
|
assert len(result) == 2
|
|
# Should detect the gap for the single missing candle
|
|
assert len(gaps) == 1
|
|
assert gaps[0] == (1709337660, 1709337660) # Single candle gap
|
|
|
|
def test_single_candle_end_gap_detection(self, db_cache):
|
|
"""Test that a single missing candle at the end is detected as a gap."""
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
db_cache.put("binance", "BTC/USDT", "1m", candles)
|
|
|
|
result, gaps = db_cache.get(
|
|
"binance", "BTC/USDT", "1m",
|
|
start=1709337600,
|
|
end=1709337780,
|
|
)
|
|
|
|
assert len(result) == 3
|
|
assert len(gaps) == 1
|
|
assert gaps[0] == (1709337780, 1709337780) # Single candle end gap
|
|
|
|
|
|
class TestMemoryCacheBinarySearch:
|
|
"""Tests for binary search optimizations in MemoryCache."""
|
|
|
|
def test_binary_search_range_query_large_dataset(self):
|
|
"""Test that time range queries work correctly on large datasets."""
|
|
cache = MemoryCache()
|
|
|
|
# Create 1000 candles (1m intervals)
|
|
candles = [
|
|
Candle(
|
|
time=1709337600 + i * 60,
|
|
open=50000.0 + i,
|
|
high=50100.0 + i,
|
|
low=49900.0 + i,
|
|
close=50050.0 + i,
|
|
volume=10.0,
|
|
)
|
|
for i in range(1000)
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
# Query a middle range
|
|
start = 1709337600 + 100 * 60 # Candle 100
|
|
end = 1709337600 + 200 * 60 # Candle 200
|
|
result, _ = cache.get("binance:BTC/USDT:1m", start=start, end=end)
|
|
|
|
assert len(result) == 101 # Inclusive range
|
|
assert result[0].time == start
|
|
assert result[-1].time == end
|
|
|
|
def test_binary_search_update_maintains_order(self):
|
|
"""Test that update_candle maintains sort order via binary insert."""
|
|
cache = MemoryCache()
|
|
|
|
# Start with some candles
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
# Insert a candle in the middle
|
|
middle_candle = Candle(time=1709337660, open=51000.0, high=51000.0, low=51000.0, close=51000.0, volume=2.0)
|
|
cache.update_candle("binance:BTC/USDT:1m", middle_candle)
|
|
|
|
result, _ = cache.get("binance:BTC/USDT:1m")
|
|
|
|
# Should be sorted
|
|
assert len(result) == 3
|
|
assert result[0].time == 1709337600
|
|
assert result[1].time == 1709337660
|
|
assert result[2].time == 1709337720
|
|
|
|
def test_binary_search_finds_exact_candle_for_update(self):
|
|
"""Test that update_candle finds and updates exact match."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337720, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
# Update middle candle
|
|
updated = Candle(time=1709337660, open=99999.0, high=99999.0, low=99999.0, close=99999.0, volume=99.0)
|
|
cache.update_candle("binance:BTC/USDT:1m", updated)
|
|
|
|
result, _ = cache.get("binance:BTC/USDT:1m")
|
|
|
|
assert len(result) == 3 # No new candle added
|
|
assert result[1].open == 99999.0
|
|
|
|
def test_binary_search_empty_range_result(self):
|
|
"""Test querying a range with no candles returns empty."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=1709337600, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
Candle(time=1709337660, open=50000.0, high=50000.0, low=50000.0, close=50000.0, volume=1.0),
|
|
]
|
|
cache.put("binance:BTC/USDT:1m", candles)
|
|
|
|
# Query range after all candles
|
|
result, gaps = cache.get("binance:BTC/USDT:1m", start=1709338000, end=1709339000)
|
|
|
|
assert len(result) == 0
|
|
assert len(gaps) == 1
|
|
|
|
def test_binary_search_boundary_conditions(self):
|
|
"""Test boundary conditions for binary search."""
|
|
cache = MemoryCache()
|
|
|
|
candles = [
|
|
Candle(time=100, open=1.0, high=1.0, low=1.0, close=1.0, volume=1.0),
|
|
Candle(time=200, open=2.0, high=2.0, low=2.0, close=2.0, volume=1.0),
|
|
Candle(time=300, open=3.0, high=3.0, low=3.0, close=3.0, volume=1.0),
|
|
]
|
|
cache.put("test:TEST:1m", candles)
|
|
|
|
# Query exact first candle
|
|
result, _ = cache.get("test:TEST:1m", start=100, end=100)
|
|
assert len(result) == 1
|
|
assert result[0].time == 100
|
|
|
|
# Query exact last candle
|
|
result, _ = cache.get("test:TEST:1m", start=300, end=300)
|
|
assert len(result) == 1
|
|
assert result[0].time == 300
|
|
|
|
# Query before all candles
|
|
result, _ = cache.get("test:TEST:1m", start=50, end=99)
|
|
assert len(result) == 0
|
|
|
|
# Query after all candles
|
|
result, _ = cache.get("test:TEST:1m", start=301, end=400)
|
|
assert len(result) == 0
|