522 lines
17 KiB
Python
522 lines
17 KiB
Python
"""
|
|
Integration tests simulating BrighterTrading client workflows.
|
|
|
|
These tests verify the Exchange Data Manager behaves correctly
|
|
for BrighterTrading's expected usage patterns:
|
|
|
|
1. Range requests with start/end timestamps
|
|
2. LAST_N mode (limit-only) requests
|
|
3. Gap-filled continuous data
|
|
4. Session-based authentication workflow
|
|
5. WebSocket subscription flow
|
|
"""
|
|
|
|
import pytest
|
|
import asyncio
|
|
from typing import List
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
import tempfile
|
|
import os
|
|
|
|
from exchange_data_manager.cache.manager import CacheManager
|
|
from exchange_data_manager.cache.async_database import AsyncDatabaseCache
|
|
from exchange_data_manager.cache.memory import MemoryCache
|
|
from exchange_data_manager.candles.models import Candle, CandleRequest
|
|
from exchange_data_manager.config import CacheConfig, DatabaseConfig
|
|
from exchange_data_manager.sessions import SessionManager
|
|
|
|
|
|
def generate_continuous_candles(
|
|
count: int,
|
|
start_time: int = 1700000000,
|
|
interval: int = 300,
|
|
) -> List[Candle]:
|
|
"""Generate continuous candles without gaps."""
|
|
return [
|
|
Candle(
|
|
time=start_time + i * interval,
|
|
open=100.0 + i * 0.01,
|
|
high=100.5 + i * 0.01,
|
|
low=99.5 + i * 0.01,
|
|
close=100.25 + i * 0.01,
|
|
volume=1000.0 + i,
|
|
closed=True,
|
|
)
|
|
for i in range(count)
|
|
]
|
|
|
|
|
|
def generate_candles_with_gaps(
|
|
count: int,
|
|
start_time: int = 1700000000,
|
|
interval: int = 300,
|
|
gap_indices: List[int] = None,
|
|
) -> List[Candle]:
|
|
"""Generate candles with specific gaps."""
|
|
gap_indices = gap_indices or []
|
|
candles = []
|
|
|
|
for i in range(count):
|
|
if i not in gap_indices:
|
|
candles.append(Candle(
|
|
time=start_time + i * interval,
|
|
open=100.0 + i * 0.01,
|
|
high=100.5 + i * 0.01,
|
|
low=99.5 + i * 0.01,
|
|
close=100.25 + i * 0.01,
|
|
volume=1000.0 + i,
|
|
closed=True,
|
|
))
|
|
|
|
return candles
|
|
|
|
|
|
class TestRangeRequests:
|
|
"""Test range request scenarios (start/end specified)."""
|
|
|
|
@pytest.fixture
|
|
def temp_db_path(self):
|
|
"""Create a temporary database path."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
yield os.path.join(tmpdir, "test.db")
|
|
|
|
@pytest.fixture
|
|
async def cache_manager(self, temp_db_path):
|
|
"""Create initialized cache manager."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(fill_gaps=True),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
yield manager
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_range_request_returns_correct_format(self, cache_manager):
|
|
"""Verify range request returns data in expected format."""
|
|
# Pre-populate with candles
|
|
candles = generate_continuous_candles(100)
|
|
cache_manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
start=candles[10].time,
|
|
end=candles[50].time,
|
|
)
|
|
|
|
result = await cache_manager.get_candles(request)
|
|
|
|
# Verify format
|
|
assert isinstance(result, list)
|
|
assert all(isinstance(c, Candle) for c in result)
|
|
|
|
# Verify time range
|
|
assert result[0].time >= request.start
|
|
assert result[-1].time <= request.end
|
|
|
|
# Verify sorted order
|
|
times = [c.time for c in result]
|
|
assert times == sorted(times)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_range_request_validates_bounds(self, cache_manager):
|
|
"""Verify range request respects start/end bounds."""
|
|
candles = generate_continuous_candles(100)
|
|
cache_manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
start_time = candles[25].time
|
|
end_time = candles[75].time
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
start=start_time,
|
|
end=end_time,
|
|
)
|
|
|
|
result = await cache_manager.get_candles(request)
|
|
|
|
# All candles should be within bounds
|
|
for candle in result:
|
|
assert candle.time >= start_time
|
|
assert candle.time <= end_time
|
|
|
|
|
|
class TestLastNRequests:
|
|
"""Test LAST_N mode (limit-only) requests."""
|
|
|
|
@pytest.fixture
|
|
def temp_db_path(self):
|
|
"""Create a temporary database path."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
yield os.path.join(tmpdir, "test.db")
|
|
|
|
@pytest.fixture
|
|
async def cache_manager(self, temp_db_path):
|
|
"""Create initialized cache manager."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(fill_gaps=True),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
yield manager
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_limit_only_request(self, cache_manager):
|
|
"""Test request with only limit specified (no start/end)."""
|
|
candles = generate_continuous_candles(100)
|
|
cache_manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
limit=20,
|
|
)
|
|
|
|
result = await cache_manager.get_candles(request)
|
|
|
|
# Should return most recent N candles
|
|
assert len(result) <= 20
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_limit_returns_most_recent(self, cache_manager):
|
|
"""Verify limit returns most recent candles."""
|
|
candles = generate_continuous_candles(100)
|
|
cache_manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
limit=10,
|
|
)
|
|
|
|
result = await cache_manager.get_candles(request)
|
|
|
|
# Should be the last 10 candles
|
|
expected_times = [c.time for c in candles[-10:]]
|
|
actual_times = [c.time for c in result]
|
|
assert actual_times == expected_times
|
|
|
|
|
|
class TestGapFilledData:
|
|
"""Test gap detection and filling."""
|
|
|
|
@pytest.fixture
|
|
def temp_db_path(self):
|
|
"""Create a temporary database path."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
yield os.path.join(tmpdir, "test.db")
|
|
|
|
@pytest.fixture
|
|
async def cache_manager_with_gap_fill(self, temp_db_path):
|
|
"""Create cache manager with gap filling enabled."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(fill_gaps=True, forward_fill_volume=True),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
yield manager
|
|
|
|
@pytest.fixture
|
|
async def cache_manager_no_gap_fill(self, temp_db_path):
|
|
"""Create cache manager with gap filling disabled."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(fill_gaps=False),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
yield manager
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_gap_filled_data_is_continuous(self, cache_manager_with_gap_fill):
|
|
"""Verify gap-filled data has no missing candles."""
|
|
# Create candles with gaps at indices 5, 10, 15
|
|
candles = generate_candles_with_gaps(20, gap_indices=[5, 10, 15])
|
|
cache_manager_with_gap_fill.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
start_time = candles[0].time
|
|
end_time = 1700000000 + 19 * 300 # Full 20-candle range
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
start=start_time,
|
|
end=end_time,
|
|
)
|
|
|
|
result = await cache_manager_with_gap_fill.get_candles(request)
|
|
|
|
# Verify continuity - no gaps
|
|
for i in range(1, len(result)):
|
|
expected_time = result[i - 1].time + 300
|
|
assert result[i].time == expected_time, f"Gap detected at index {i}"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_gap_fill_uses_forward_fill(self, cache_manager_with_gap_fill):
|
|
"""Verify filled candles use next candle's OHLCV (forward-fill strategy)."""
|
|
candles = generate_candles_with_gaps(10, gap_indices=[5])
|
|
cache_manager_with_gap_fill.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
start_time = candles[0].time
|
|
end_time = 1700000000 + 9 * 300
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
start=start_time,
|
|
end=end_time,
|
|
)
|
|
|
|
result = await cache_manager_with_gap_fill.get_candles(request)
|
|
|
|
# Find the filled candle (at index 5)
|
|
filled_time = 1700000000 + 5 * 300
|
|
filled_candle = next((c for c in result if c.time == filled_time), None)
|
|
|
|
assert filled_candle is not None, "Filled candle should exist"
|
|
|
|
# Next candle after the gap (index 6, time = 1700001800)
|
|
next_candle_time = 1700000000 + 6 * 300
|
|
next_candle = next((c for c in result if c.time == next_candle_time), None)
|
|
assert next_candle is not None
|
|
|
|
# Forward-fill: filled candle should use NEXT candle's values
|
|
assert filled_candle.open == next_candle.open
|
|
assert filled_candle.high == next_candle.high
|
|
assert filled_candle.low == next_candle.low
|
|
assert filled_candle.close == next_candle.close
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_gap_fill_preserves_gaps(self, cache_manager_no_gap_fill):
|
|
"""Verify gaps are preserved when fill_gaps=False."""
|
|
candles = generate_candles_with_gaps(10, gap_indices=[5])
|
|
cache_manager_no_gap_fill.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
limit=20,
|
|
)
|
|
|
|
result = await cache_manager_no_gap_fill.get_candles(request)
|
|
|
|
# Should have 9 candles (original 10 minus 1 gap)
|
|
assert len(result) == 9
|
|
|
|
|
|
class TestSessionWorkflow:
|
|
"""Test session-based authentication workflow."""
|
|
|
|
@pytest.fixture
|
|
async def session_manager(self):
|
|
"""Create session manager."""
|
|
manager = SessionManager(
|
|
session_timeout_minutes=60,
|
|
cleanup_interval_seconds=300,
|
|
)
|
|
await manager.start()
|
|
yield manager
|
|
await manager.stop()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_create_credentials_fetch_destroy(self, session_manager):
|
|
"""Test full session lifecycle."""
|
|
# 1. Create session
|
|
session = session_manager.create_session()
|
|
assert session is not None
|
|
assert session.id is not None
|
|
|
|
# 2. Add credentials (no mock needed - SessionManager stores credentials directly)
|
|
success = await session_manager.add_exchange_credentials(
|
|
session_id=session.id,
|
|
exchange="binance",
|
|
api_key="test_key",
|
|
api_secret="test_secret",
|
|
)
|
|
assert success
|
|
|
|
# 3. Verify session has exchange
|
|
session_info = session_manager.get_session(session.id)
|
|
assert session_info is not None
|
|
assert "binance" in session_info.exchanges
|
|
|
|
# 4. Destroy session
|
|
destroyed = await session_manager.destroy_session(session.id)
|
|
assert destroyed
|
|
|
|
# 5. Verify session is gone
|
|
session_info = session_manager.get_session(session.id)
|
|
assert session_info is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_refresh_extends_expiry(self, session_manager):
|
|
"""Test that refreshing session extends expiry."""
|
|
session = session_manager.create_session()
|
|
original_expiry = session.expires_at
|
|
|
|
# Small delay to ensure time passes
|
|
await asyncio.sleep(0.1)
|
|
|
|
# Refresh
|
|
success = session_manager.refresh_session(session.id)
|
|
assert success
|
|
|
|
# Check new expiry is later
|
|
refreshed_session = session_manager.get_session(session.id)
|
|
assert refreshed_session.expires_at > original_expiry
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_session_stats(self, session_manager):
|
|
"""Test session statistics."""
|
|
# Create some sessions
|
|
session_manager.create_session()
|
|
session_manager.create_session()
|
|
|
|
stats = session_manager.stats()
|
|
|
|
assert "total_sessions" in stats
|
|
assert "active_sessions" in stats
|
|
assert stats["total_sessions"] >= 2
|
|
assert stats["active_sessions"] >= 2
|
|
|
|
|
|
class TestWebSocketFlow:
|
|
"""Test WebSocket subscription flow."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_subscription_message_format(self):
|
|
"""Verify subscription message format is correct."""
|
|
from exchange_data_manager.api.websocket import ws_manager
|
|
|
|
# Create a mock websocket
|
|
mock_ws = AsyncMock()
|
|
mock_ws.send_json = AsyncMock()
|
|
|
|
# Simulate connection
|
|
await ws_manager.connect(mock_ws)
|
|
|
|
try:
|
|
# Simulate subscription message
|
|
subscribe_msg = '{"action": "subscribe", "exchange": "binance", "symbol": "BTC/USDT", "timeframe": "1m"}'
|
|
|
|
# Note: This would need the full infrastructure to test properly
|
|
# Here we just verify the message format is parseable
|
|
import json
|
|
parsed = json.loads(subscribe_msg)
|
|
|
|
assert parsed["action"] == "subscribe"
|
|
assert parsed["exchange"] == "binance"
|
|
assert parsed["symbol"] == "BTC/USDT"
|
|
assert parsed["timeframe"] == "1m"
|
|
finally:
|
|
await ws_manager.disconnect(mock_ws)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ping_pong(self):
|
|
"""Verify ping/pong keep-alive works."""
|
|
import json
|
|
|
|
ping_msg = '{"action": "ping"}'
|
|
parsed = json.loads(ping_msg)
|
|
assert parsed["action"] == "ping"
|
|
|
|
|
|
class TestDataIntegrity:
|
|
"""Test data integrity guarantees."""
|
|
|
|
@pytest.fixture
|
|
def temp_db_path(self):
|
|
"""Create a temporary database path."""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
yield os.path.join(tmpdir, "test.db")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_candle_timestamps_are_consistent(self, temp_db_path):
|
|
"""Verify candle timestamps are evenly spaced by timeframe interval."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
|
|
# Use a start time that's aligned to 5m (divisible by 300)
|
|
aligned_start = 1700000100 - (1700000100 % 300) # 1699999800
|
|
candles = generate_continuous_candles(10, start_time=aligned_start, interval=300)
|
|
manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
limit=10,
|
|
)
|
|
|
|
result = await manager.get_candles(request)
|
|
|
|
# Verify timestamps are evenly spaced
|
|
for i in range(1, len(result)):
|
|
diff = result[i].time - result[i-1].time
|
|
assert diff == 300, f"Candle spacing is {diff}s, expected 300s"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_duplicate_candles_deduplicated(self, temp_db_path):
|
|
"""Verify duplicate candles are deduplicated."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
|
|
candles = generate_continuous_candles(10)
|
|
|
|
# Put same candles twice
|
|
manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
manager.memory.put("binance:BTC/USDT:5m", candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
limit=20,
|
|
)
|
|
|
|
result = await manager.get_candles(request)
|
|
|
|
# Should only have 10 candles, not 20
|
|
assert len(result) == 10
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_candles_sorted_ascending(self, temp_db_path):
|
|
"""Verify candles are always returned in ascending time order."""
|
|
manager = CacheManager(
|
|
cache_config=CacheConfig(),
|
|
database_config=DatabaseConfig(path=temp_db_path),
|
|
)
|
|
await manager.initialize()
|
|
|
|
# Create candles and add in reverse order
|
|
candles = generate_continuous_candles(10)
|
|
reversed_candles = list(reversed(candles))
|
|
|
|
manager.memory.put("binance:BTC/USDT:5m", reversed_candles)
|
|
|
|
request = CandleRequest(
|
|
exchange="binance",
|
|
symbol="BTC/USDT",
|
|
timeframe="5m",
|
|
limit=10,
|
|
)
|
|
|
|
result = await manager.get_candles(request)
|
|
|
|
# Should be sorted ascending
|
|
times = [c.time for c in result]
|
|
assert times == sorted(times)
|