"""
External Indicators Manager - Handles custom API-based indicators with historical data.

Allows users to define external data sources that provide historical data,
enabling their use in backtesting alongside standard technical indicators.
"""

import json
import logging
import uuid
import datetime as dt
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple
from datetime import date, datetime
import requests
from jsonpath_ng import parse as jsonpath_parse
import pandas as pd

from DataCache_v3 import DataCache

logger = logging.getLogger(__name__)


@dataclass
class ExternalIndicatorConfig:
    """Configuration for an external API-based indicator."""
    name: str
    historical_url: str  # URL template with {start_date}, {end_date}, {limit} placeholders
    auth_header: str = ""  # e.g., "X-CMC_PRO_API_KEY"
    auth_key: str = ""  # The actual API key
    value_jsonpath: str = "$.data[*].value"  # JSONPath to extract values array
    timestamp_jsonpath: str = "$.data[*].timestamp"  # JSONPath to extract timestamps
    date_format: str = "ISO"  # "ISO" (2024-01-15) or "UNIX" (timestamp)
    date_param_format: str = "ISO"  # Format for URL date parameters
    creator_id: int = None
    enabled: bool = True
    tbl_key: str = None

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return {
            'name': self.name,
            'historical_url': self.historical_url,
            'auth_header': self.auth_header,
            'value_jsonpath': self.value_jsonpath,
            'timestamp_jsonpath': self.timestamp_jsonpath,
            'date_format': self.date_format,
            'date_param_format': self.date_param_format,
            'creator_id': self.creator_id,
            'enabled': self.enabled,
            'tbl_key': self.tbl_key,
            # Note: auth_key intentionally excluded for security
        }


class ExternalIndicator:
    """
    Represents an external indicator instance with cached historical data.
    """

    def __init__(self, config: ExternalIndicatorConfig, data_cache: DataCache):
        self.config = config
        self.data_cache = data_cache
        self._cache: Dict[date, float] = {}  # In-memory cache: date -> value
        self._cache_loaded = False

    def _load_cache_from_db(self) -> None:
        """Load cached historical data from database."""
        if self._cache_loaded:
            return

        try:
            rows = self.data_cache.db.execute_sql(
                """SELECT data_date, value FROM external_indicator_data
                   WHERE indicator_key = ? ORDER BY data_date""",
                params=(self.config.tbl_key,),
                fetch_all=True
            )
            if rows:
                for row in rows:
                    data_date = datetime.strptime(row[0], '%Y-%m-%d').date()
                    self._cache[data_date] = float(row[1])
                logger.debug(f"Loaded {len(self._cache)} cached values for {self.config.name}")
            self._cache_loaded = True
        except Exception as e:
            logger.error(f"Error loading indicator cache: {e}")
            self._cache_loaded = True

    def _save_to_cache(self, data_date: date, value: float) -> None:
        """Save a single data point to database cache."""
        try:
            self.data_cache.db.execute_sql(
                """INSERT OR REPLACE INTO external_indicator_data
                   (indicator_key, data_date, value) VALUES (?, ?, ?)""",
                params=(self.config.tbl_key, data_date.strftime('%Y-%m-%d'), value)
            )
        except Exception as e:
            logger.error(f"Error saving to indicator cache: {e}")

    def _format_date_param(self, d: date) -> str:
        """Format a date for URL parameter based on config."""
        if self.config.date_param_format == "UNIX":
            return str(int(datetime.combine(d, datetime.min.time()).timestamp()))
        else:  # ISO
            return d.strftime('%Y-%m-%d')

    def _parse_timestamp(self, ts_value: Any) -> Optional[date]:
        """Parse a timestamp from API response."""
        try:
            if self.config.date_format == "UNIX":
                # Unix timestamp (seconds or milliseconds)
                ts = float(ts_value)
                if ts > 1e11:  # Milliseconds
                    ts = ts / 1000
                return datetime.fromtimestamp(ts).date()
            else:  # ISO format
                if isinstance(ts_value, str):
                    # Try common formats
                    for fmt in ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ',
                                '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d']:
                        try:
                            return datetime.strptime(ts_value[:19], fmt[:len(ts_value)]).date()
                        except ValueError:
                            continue
                    # Last resort: just take first 10 chars as date
                    return datetime.strptime(ts_value[:10], '%Y-%m-%d').date()
            return None
        except Exception as e:
            logger.debug(f"Error parsing timestamp {ts_value}: {e}")
            return None

    def fetch_historical(self, start_date: date, end_date: date) -> Dict[date, float]:
        """
        Fetch historical data from API for the given date range.

        :param start_date: Start of date range.
        :param end_date: End of date range.
        :return: Dict mapping dates to values.
        """
        self._load_cache_from_db()

        # Check what we already have cached
        missing_start = start_date
        missing_end = end_date

        # Find gaps in cache
        cached_dates = set(self._cache.keys())
        requested_dates = set()
        current = start_date
        while current <= end_date:
            requested_dates.add(current)
            current += dt.timedelta(days=1)

        missing_dates = requested_dates - cached_dates

        if not missing_dates:
            # All data is cached
            return {d: self._cache[d] for d in requested_dates if d in self._cache}

        # Need to fetch missing data
        missing_start = min(missing_dates)
        missing_end = max(missing_dates)

        try:
            # Build URL with date parameters
            url = self.config.historical_url
            url = url.replace('{start_date}', self._format_date_param(missing_start))
            url = url.replace('{end_date}', self._format_date_param(missing_end))

            # Calculate limit (days between dates)
            limit = (missing_end - missing_start).days + 1
            url = url.replace('{limit}', str(limit))

            headers = {'Accept': 'application/json'}
            if self.config.auth_header and self.config.auth_key:
                headers[self.config.auth_header] = self.config.auth_key

            response = requests.get(url, headers=headers, timeout=30)

            if not response.ok:
                logger.error(f"API error fetching {self.config.name}: {response.status_code}")
                return {d: self._cache.get(d) for d in requested_dates if d in self._cache}

            data = response.json()

            logger.info(f"[ExternalIndicator] API Response keys: {list(data.keys()) if isinstance(data, dict) else 'not a dict'}")

            # Extract values and timestamps using JSONPath
            value_expr = jsonpath_parse(self.config.value_jsonpath)
            timestamp_expr = jsonpath_parse(self.config.timestamp_jsonpath)

            values = [match.value for match in value_expr.find(data)]
            timestamps = [match.value for match in timestamp_expr.find(data)]

            logger.info(f"[ExternalIndicator] Extracted {len(values)} values and {len(timestamps)} timestamps")
            if values:
                logger.info(f"[ExternalIndicator] Sample values: {values[:3]}")
            if timestamps:
                logger.info(f"[ExternalIndicator] Sample timestamps: {timestamps[:3]}")

            if len(values) != len(timestamps):
                logger.error(f"Mismatch: {len(values)} values, {len(timestamps)} timestamps")
                return {d: self._cache.get(d) for d in requested_dates if d in self._cache}

            # Parse and cache the data
            for ts, val in zip(timestamps, values):
                parsed_date = self._parse_timestamp(ts)
                if parsed_date:
                    try:
                        numeric_val = float(val)
                        self._cache[parsed_date] = numeric_val
                        self._save_to_cache(parsed_date, numeric_val)
                    except (TypeError, ValueError):
                        logger.debug(f"Non-numeric value for {parsed_date}: {val}")

            logger.info(f"Fetched {len(values)} data points for {self.config.name}")

        except requests.exceptions.RequestException as e:
            logger.error(f"Request error fetching {self.config.name}: {e}")
        except Exception as e:
            logger.error(f"Error fetching {self.config.name}: {e}", exc_info=True)

        # Return requested data from cache
        return {d: self._cache.get(d) for d in requested_dates if d in self._cache}

    def get_value_for_timestamp(self, ts: datetime) -> Optional[float]:
        """
        Get the indicator value for a specific timestamp.
        Uses the value for that day (handles granularity mismatch).

        :param ts: The timestamp to look up.
        :return: The indicator value or None if not available.
        """
        self._load_cache_from_db()
        target_date = ts.date() if isinstance(ts, datetime) else ts
        return self._cache.get(target_date)

    def calculate(self, candles: pd.DataFrame) -> pd.Series:
        """
        Calculate indicator values for a DataFrame of candles.

        :param candles: DataFrame with 'time', 'timestamp', or 'date' column.
        :return: Series of indicator values aligned to candles.
        """
        if candles.empty:
            return pd.Series(dtype=float)

        # Determine date range from candles
        # Check various column names used by different data sources
        if 'time' in candles.columns:
            # 'time' column is typically Unix timestamp in milliseconds
            time_col = candles['time']
            if time_col.dtype in ['int64', 'float64'] and time_col.iloc[0] > 1e9:
                # Convert from milliseconds or seconds
                if time_col.iloc[0] > 1e12:
                    dates = pd.to_datetime(time_col, unit='ms')
                else:
                    dates = pd.to_datetime(time_col, unit='s')
            else:
                dates = pd.to_datetime(time_col)
        elif 'timestamp' in candles.columns:
            dates = pd.to_datetime(candles['timestamp'])
        elif 'date' in candles.columns:
            dates = pd.to_datetime(candles['date'])
        elif 'datetime' in candles.columns:
            dates = pd.to_datetime(candles['datetime'])
        else:
            # Try index
            dates = pd.to_datetime(candles.index)

        start_date = dates.min().date()
        end_date = dates.max().date()

        logger.info(f"[ExternalIndicator] Fetching data for {self.config.name} from {start_date} to {end_date}")

        # Fetch historical data for this range
        historical_data = self.fetch_historical(start_date, end_date)

        logger.info(f"[ExternalIndicator] Got {len(historical_data)} data points from API")

        # Map each candle to its date's value
        values = []
        for d in dates:
            candle_date = d.date()
            val = historical_data.get(candle_date)
            values.append(val)

        # Log some stats
        non_null = sum(1 for v in values if v is not None)
        logger.info(f"[ExternalIndicator] Mapped {non_null}/{len(values)} candles to values")

        return pd.Series(values, index=candles.index, name=self.config.name)


class ExternalIndicatorsManager:
    """Manages external API-based indicators."""

    def __init__(self, data_cache: DataCache):
        self.data_cache = data_cache
        self._ensure_tables_exist()

        # In-memory storage of indicator configs
        self.configs: Dict[str, ExternalIndicatorConfig] = {}

        # Cached ExternalIndicator instances
        self._indicators: Dict[str, ExternalIndicator] = {}

        # Load existing configs from database
        self._load_configs_from_db()

    def _ensure_tables_exist(self) -> None:
        """Create necessary database tables."""
        try:
            # Table for indicator configurations
            if not self.data_cache.db.table_exists('external_indicators'):
                self.data_cache.db.execute_sql("""
                    CREATE TABLE IF NOT EXISTS external_indicators (
                        id INTEGER PRIMARY KEY AUTOINCREMENT,
                        tbl_key TEXT UNIQUE NOT NULL,
                        creator_id INTEGER,
                        name TEXT NOT NULL,
                        historical_url TEXT NOT NULL,
                        auth_header TEXT,
                        auth_key TEXT,
                        value_jsonpath TEXT DEFAULT '$.data[*].value',
                        timestamp_jsonpath TEXT DEFAULT '$.data[*].timestamp',
                        date_format TEXT DEFAULT 'ISO',
                        date_param_format TEXT DEFAULT 'ISO',
                        enabled INTEGER DEFAULT 1
                    )
                """, params=[])
                logger.info("Created external_indicators table")

            # Table for cached historical data
            if not self.data_cache.db.table_exists('external_indicator_data'):
                self.data_cache.db.execute_sql("""
                    CREATE TABLE IF NOT EXISTS external_indicator_data (
                        id INTEGER PRIMARY KEY AUTOINCREMENT,
                        indicator_key TEXT NOT NULL,
                        data_date TEXT NOT NULL,
                        value REAL NOT NULL,
                        UNIQUE(indicator_key, data_date)
                    )
                """, params=[])
                self.data_cache.db.execute_sql(
                    "CREATE INDEX IF NOT EXISTS idx_ext_ind_data ON external_indicator_data(indicator_key, data_date)",
                    params=[]
                )
                logger.info("Created external_indicator_data table")

        except Exception as e:
            logger.error(f"Error creating external indicator tables: {e}", exc_info=True)

    def _load_configs_from_db(self) -> None:
        """Load all indicator configurations from database."""
        try:
            rows = self.data_cache.db.execute_sql(
                "SELECT * FROM external_indicators",
                params=[],
                fetch_all=True
            )
            if rows:
                # Get column names
                column_info = self.data_cache.db.execute_sql(
                    "PRAGMA table_info(external_indicators)",
                    params=[],
                    fetch_all=True
                )
                columns = [row[1] for row in (column_info or [])]

                for row in rows:
                    row_dict = dict(zip(columns, row))
                    config = ExternalIndicatorConfig(
                        name=row_dict.get('name', ''),
                        historical_url=row_dict.get('historical_url', ''),
                        auth_header=row_dict.get('auth_header', ''),
                        auth_key=row_dict.get('auth_key', ''),
                        value_jsonpath=row_dict.get('value_jsonpath', '$.data[*].value'),
                        timestamp_jsonpath=row_dict.get('timestamp_jsonpath', '$.data[*].timestamp'),
                        date_format=row_dict.get('date_format', 'ISO'),
                        date_param_format=row_dict.get('date_param_format', 'ISO'),
                        creator_id=row_dict.get('creator_id'),
                        enabled=bool(row_dict.get('enabled', 1)),
                        tbl_key=row_dict.get('tbl_key')
                    )
                    if config.tbl_key:
                        self.configs[config.tbl_key] = config
                logger.info(f"Loaded {len(self.configs)} external indicator configs")
        except Exception as e:
            logger.error(f"Error loading external indicator configs: {e}", exc_info=True)

    def _save_config_to_db(self, config: ExternalIndicatorConfig) -> None:
        """Save an indicator configuration to database."""
        existing = self.data_cache.db.execute_sql(
            "SELECT id FROM external_indicators WHERE tbl_key = ?",
            params=(config.tbl_key,),
            fetch_one=True
        )

        if existing:
            self.data_cache.db.execute_sql("""
                UPDATE external_indicators SET
                    creator_id = ?, name = ?, historical_url = ?, auth_header = ?,
                    auth_key = ?, value_jsonpath = ?, timestamp_jsonpath = ?,
                    date_format = ?, date_param_format = ?, enabled = ?
                WHERE tbl_key = ?
            """, params=(
                config.creator_id, config.name, config.historical_url,
                config.auth_header, config.auth_key, config.value_jsonpath,
                config.timestamp_jsonpath, config.date_format, config.date_param_format,
                1 if config.enabled else 0, config.tbl_key
            ))
        else:
            self.data_cache.db.execute_sql("""
                INSERT INTO external_indicators (
                    tbl_key, creator_id, name, historical_url, auth_header, auth_key,
                    value_jsonpath, timestamp_jsonpath, date_format, date_param_format, enabled
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, params=(
                config.tbl_key, config.creator_id, config.name, config.historical_url,
                config.auth_header, config.auth_key, config.value_jsonpath,
                config.timestamp_jsonpath, config.date_format, config.date_param_format,
                1 if config.enabled else 0
            ))

    def test_indicator(self, historical_url: str, auth_header: str, auth_key: str,
                       value_jsonpath: str, timestamp_jsonpath: str,
                       date_format: str, date_param_format: str) -> Dict[str, Any]:
        """
        Test an external indicator configuration by fetching sample data.

        :return: Dict with success status, sample values, and any errors.
        """
        try:
            # Build URL with sample date range (last 7 days)
            end_date = date.today()
            start_date = end_date - dt.timedelta(days=7)

            url = historical_url

            # Format dates based on config
            if date_param_format == "UNIX":
                start_str = str(int(datetime.combine(start_date, datetime.min.time()).timestamp()))
                end_str = str(int(datetime.combine(end_date, datetime.min.time()).timestamp()))
            else:
                start_str = start_date.strftime('%Y-%m-%d')
                end_str = end_date.strftime('%Y-%m-%d')

            url = url.replace('{start_date}', start_str)
            url = url.replace('{end_date}', end_str)
            url = url.replace('{limit}', '7')

            headers = {'Accept': 'application/json'}
            if auth_header and auth_key:
                headers[auth_header] = auth_key

            response = requests.get(url, headers=headers, timeout=15)

            # Try to parse JSON for error messages
            try:
                data = response.json()
            except json.JSONDecodeError:
                return {
                    'success': False,
                    'error': f'Invalid JSON response (HTTP {response.status_code})'
                }

            if not response.ok:
                # Try to extract API error message
                error_msg = self._extract_api_error(data)
                if error_msg:
                    return {'success': False, 'error': f'API Error: {error_msg}'}
                return {'success': False, 'error': f'HTTP {response.status_code}: {response.reason}'}

            # Extract values using JSONPath
            value_expr = jsonpath_parse(value_jsonpath)
            timestamp_expr = jsonpath_parse(timestamp_jsonpath)

            values = [match.value for match in value_expr.find(data)]
            timestamps = [match.value for match in timestamp_expr.find(data)]

            if not values:
                return {
                    'success': False,
                    'error': f"Value JSONPath '{value_jsonpath}' found no matches",
                    'response_preview': str(data)[:300]
                }

            if not timestamps:
                return {
                    'success': False,
                    'error': f"Timestamp JSONPath '{timestamp_jsonpath}' found no matches",
                    'response_preview': str(data)[:300]
                }

            if len(values) != len(timestamps):
                return {
                    'success': False,
                    'error': f"Mismatch: {len(values)} values but {len(timestamps)} timestamps"
                }

            # Verify values are numeric
            sample_values = []
            for val in values[:5]:
                try:
                    sample_values.append(float(val))
                except (TypeError, ValueError):
                    return {
                        'success': False,
                        'error': f"Non-numeric value found: {val}"
                    }

            # Verify timestamps can be parsed
            parsed_dates = []
            for ts in timestamps[:5]:
                parsed = self._parse_timestamp_static(ts, date_format)
                if parsed:
                    parsed_dates.append(parsed.strftime('%Y-%m-%d'))
                else:
                    return {
                        'success': False,
                        'error': f"Could not parse timestamp: {ts}"
                    }

            return {
                'success': True,
                'data_points': len(values),
                'sample_values': sample_values,
                'sample_dates': parsed_dates,
                'message': f"Found {len(values)} data points"
            }

        except requests.exceptions.Timeout:
            return {'success': False, 'error': 'Request timed out (15s)'}
        except requests.exceptions.ConnectionError:
            return {'success': False, 'error': 'Connection failed - check URL'}
        except Exception as e:
            logger.error(f"Error testing external indicator: {e}", exc_info=True)
            return {'success': False, 'error': str(e)}

    def _extract_api_error(self, data: Any) -> Optional[str]:
        """Extract error message from common API error formats."""
        if not isinstance(data, dict):
            return None

        # CoinMarketCap format
        if 'status' in data and isinstance(data['status'], dict):
            return data['status'].get('error_message')

        # Generic formats
        for key in ['error', 'message', 'error_message', 'msg']:
            if key in data:
                return str(data[key])

        return None

    @staticmethod
    def _parse_timestamp_static(ts_value: Any, date_format: str) -> Optional[date]:
        """Static method to parse timestamp for testing."""
        try:
            if date_format == "UNIX":
                ts = float(ts_value)
                if ts > 1e11:
                    ts = ts / 1000
                return datetime.fromtimestamp(ts).date()
            else:
                if isinstance(ts_value, str):
                    for fmt in ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ',
                                '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d']:
                        try:
                            return datetime.strptime(ts_value[:19], fmt[:len(ts_value)]).date()
                        except ValueError:
                            continue
                    return datetime.strptime(ts_value[:10], '%Y-%m-%d').date()
            return None
        except Exception:
            return None

    def create_indicator(self, name: str, historical_url: str, auth_header: str,
                         auth_key: str, value_jsonpath: str, timestamp_jsonpath: str,
                         date_format: str, date_param_format: str,
                         creator_id: int) -> Dict[str, Any]:
        """
        Create a new external indicator after validating the configuration.

        :return: Dict with success status and indicator data.
        """
        # First test the configuration
        test_result = self.test_indicator(
            historical_url, auth_header, auth_key,
            value_jsonpath, timestamp_jsonpath,
            date_format, date_param_format
        )

        if not test_result['success']:
            return {
                'success': False,
                'message': f"Validation failed: {test_result['error']}"
            }

        try:
            tbl_key = str(uuid.uuid4())

            config = ExternalIndicatorConfig(
                name=name,
                historical_url=historical_url,
                auth_header=auth_header,
                auth_key=auth_key,
                value_jsonpath=value_jsonpath,
                timestamp_jsonpath=timestamp_jsonpath,
                date_format=date_format,
                date_param_format=date_param_format,
                creator_id=creator_id,
                enabled=True,
                tbl_key=tbl_key
            )

            # Save to database
            self._save_config_to_db(config)

            # Add to memory
            self.configs[tbl_key] = config

            logger.info(f"Created external indicator: {name}")

            return {
                'success': True,
                'indicator': config.to_dict(),
                'test_data': test_result,
                'message': f"Created indicator '{name}' with {test_result['data_points']} data points available"
            }

        except Exception as e:
            logger.error(f"Error creating external indicator: {e}", exc_info=True)
            return {'success': False, 'message': str(e)}

    def get_indicator(self, tbl_key: str) -> Optional[ExternalIndicator]:
        """Get an ExternalIndicator instance by tbl_key."""
        if tbl_key not in self.configs:
            return None

        if tbl_key not in self._indicators:
            self._indicators[tbl_key] = ExternalIndicator(
                self.configs[tbl_key], self.data_cache
            )

        return self._indicators[tbl_key]

    def get_indicator_by_name(self, name: str, user_id: int) -> Optional[ExternalIndicator]:
        """Get an ExternalIndicator by name for a specific user."""
        for config in self.configs.values():
            if config.name == name and config.creator_id == user_id:
                return self.get_indicator(config.tbl_key)
        return None

    def get_indicators_for_user(self, user_id: int) -> List[Dict[str, Any]]:
        """Get all indicator configs for a user."""
        return [
            config.to_dict()
            for config in self.configs.values()
            if config.creator_id == user_id
        ]

    def get_all_indicators(self) -> List[Dict[str, Any]]:
        """Get all indicator configs."""
        return [config.to_dict() for config in self.configs.values()]

    def delete_indicator(self, tbl_key: str, user_id: int) -> Dict[str, Any]:
        """Delete an external indicator."""
        config = self.configs.get(tbl_key)
        if not config:
            return {'success': False, 'message': 'Indicator not found'}

        if config.creator_id != user_id:
            return {'success': False, 'message': 'Not authorized to delete this indicator'}

        try:
            # Delete cached data
            self.data_cache.db.execute_sql(
                "DELETE FROM external_indicator_data WHERE indicator_key = ?",
                params=(tbl_key,)
            )

            # Delete config
            self.data_cache.db.execute_sql(
                "DELETE FROM external_indicators WHERE tbl_key = ?",
                params=(tbl_key,)
            )

            # Remove from memory
            del self.configs[tbl_key]
            if tbl_key in self._indicators:
                del self._indicators[tbl_key]

            logger.info(f"Deleted external indicator: {config.name}")
            return {'success': True, 'message': f"Deleted indicator '{config.name}'"}

        except Exception as e:
            logger.error(f"Error deleting external indicator: {e}", exc_info=True)
            return {'success': False, 'message': str(e)}

    def get_indicators_as_indicator_list(self, user_id: int) -> Dict[str, Dict[str, Any]]:
        """
        Get external indicators formatted for the indicator dropdown.

        :param user_id: The user ID.
        :return: Dict mapping indicator names to indicator-like data.
        """
        result = {}
        for config in self.configs.values():
            if config.creator_id == user_id and config.enabled:
                result[config.name] = {
                    'type': 'EXTERNAL_HISTORICAL',
                    'source_type': 'external_indicator',
                    'tbl_key': config.tbl_key,
                    'value': None  # Will be filled during calculation
                }
        return result
