import logging
import pandas as pd
from DataCache_v3 import DataCache
from indicators import Indicators
from trade import Trades
import datetime as dt
import json
import uuid
import traceback
from typing import Any, Optional
from PythonGenerator import PythonGenerator
from StrategyInstance import StrategyInstance
from brokers import TradingMode

# Configure logging
logger = logging.getLogger(__name__)

class Strategies:
    def __init__(self, data_cache: DataCache, trades: Trades, indicators: Indicators, edm_client=None):
        """
        Initializes the Strategies class.

        :param data_cache: Instance of DataCache to manage cache and database interactions.
        :param trades: Reference to the trades object that maintains trading actions and data.
        :param indicators: Reference to the Indicators manager.
        :param edm_client: Reference to the EDM client for candle data.
        """
        self.data_cache = data_cache  # Database interaction instance
        self.trades = trades
        self.indicators_manager = indicators
        self.edm_client = edm_client

        # Create a cache for strategies with necessary columns
        self.data_cache.create_cache(name='strategies',
                                     cache_type='table',
                                     size_limit=500,
                                     eviction_policy='deny',
                                     default_expiration=dt.timedelta(hours=24),
                                     columns=["creator", "name", "workspace", "code", "stats", "public", "fee",
                                              "tbl_key", "strategy_components", "default_source"])

        # Migration: Add 'default_source' column to existing strategies table if it doesn't exist
        self._migrate_add_default_source_column()

        # Create a cache for strategy contexts to store strategy states and settings
        self.data_cache.create_cache(
            name='strategy_contexts',
            cache_type='table',
            size_limit=1000,
            eviction_policy='deny',
            default_expiration=dt.timedelta(hours=24),
            columns=[
                "strategy_instance_id",  # Unique identifier for the strategy instance
                "flags",                 # JSON-encoded string to store flags
                "variables",
                "profit_loss",           # Float value for tracking profit/loss
                "active",                # Boolean or Integer (1/0) for active status
                "paused",                # Boolean or Integer (1/0) for paused status
                "exit",                  # Boolean or Integer (1/0) for exit status
                "exit_method",           # String defining exit method
                "start_time"             # ISO-formatted datetime string for start time
            ]
        )

        # Create a cache for strategy subscriptions
        self.data_cache.create_cache(
            name='strategy_subscriptions',
            cache_type='table',
            size_limit=1000,
            eviction_policy='deny',
            default_expiration=dt.timedelta(hours=24),
            columns=["id", "user_id", "strategy_tbl_key", "subscribed_at"]
        )

        # Ensure the subscriptions table exists in SQLite
        self._ensure_subscriptions_table()

        # Initialize default settings
        self.default_timeframe = '5m'
        self.default_exchange = 'Binance'
        self.default_symbol = 'BTCUSD'

        self.active_instances: dict[tuple[int, str, str], StrategyInstance] = {}  # Key: (user_id, strategy_id, mode)

    def _migrate_add_default_source_column(self) -> None:
        """
        Migration: Add 'default_source' column to existing strategies table if it doesn't exist.
        This handles databases created before default_source was added.
        """
        import sqlite3
        import config

        try:
            # Connect directly to the database file
            conn = sqlite3.connect(config.DB_FILE)
            cursor = conn.cursor()

            # Check if the column exists
            cursor.execute("PRAGMA table_info(strategies)")
            columns = [col[1] for col in cursor.fetchall()]

            if 'default_source' not in columns:
                logger.info("Migrating strategies table: adding 'default_source' column")
                cursor.execute("ALTER TABLE strategies ADD COLUMN default_source TEXT")
                conn.commit()
                logger.info("Migration complete: 'default_source' column added")

            conn.close()

        except Exception as e:
            logger.warning(f"Migration check failed (may be expected on fresh install): {e}")

    def _ensure_subscriptions_table(self) -> None:
        """
        Create the strategy_subscriptions table if it doesn't exist.

        Note: execute_sql runs single statements only, so we run each statement separately.
        We don't rely on FK ON DELETE CASCADE since SQLite requires PRAGMA foreign_keys = ON.
        """
        import config

        try:
            db = self.data_cache.db

            # Statement 1: Create table
            db.execute_sql("""
                CREATE TABLE IF NOT EXISTS strategy_subscriptions (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    user_id INTEGER NOT NULL,
                    strategy_tbl_key TEXT NOT NULL,
                    subscribed_at TEXT NOT NULL,
                    UNIQUE(user_id, strategy_tbl_key)
                )
            """)

            # Statement 2: Create user index
            db.execute_sql("""
                CREATE INDEX IF NOT EXISTS idx_subscriptions_user
                ON strategy_subscriptions(user_id)
            """)

            # Statement 3: Create strategy index
            db.execute_sql("""
                CREATE INDEX IF NOT EXISTS idx_subscriptions_strategy
                ON strategy_subscriptions(strategy_tbl_key)
            """)

            logger.info("Strategy subscriptions table initialized")

        except Exception as e:
            logger.warning(f"Failed to create subscriptions table (may already exist): {e}")

    def update(self, candle_data: dict = None) -> list:
        """
        Update all active strategy instances with new price data.

        Called on each candle/price tick to process strategy logic.

        :param candle_data: Optional candle data dict with 'symbol', 'close', etc.
        :return: List of all events from all strategies.
        """
        all_events = []

        if not self.active_instances:
            return all_events

        # Create a list of keys to iterate (avoid dict modification during iteration)
        instance_keys = list(self.active_instances.keys())

        for instance_key in instance_keys:
            user_id, strategy_id, mode = instance_key

            try:
                instance = self.active_instances.get(instance_key)
                if instance is None:
                    continue

                # Execute strategy tick (instance handles its own price update path).
                events = instance.tick(candle_data)

                # Tag events with instance info
                for event in events:
                    event['user_id'] = user_id
                    event['strategy_id'] = strategy_id
                    event['mode'] = mode

                all_events.extend(events)

                # Handle exit condition
                if instance.exit:
                    # Check if strategy has exited all positions
                    if hasattr(instance, 'broker'):
                        positions = instance.broker.get_all_positions()
                        if not positions:
                            logger.info(f"Strategy '{strategy_id}' has exited all positions. Removing from active.")
                            del self.active_instances[instance_key]
                            all_events.append({
                                'type': 'strategy_exited',
                                'user_id': user_id,
                                'strategy_id': strategy_id,
                                'mode': mode,
                            })

            except Exception as e:
                logger.error(f"Error updating strategy {instance_key}: {e}", exc_info=True)
                all_events.append({
                    'type': 'error',
                    'user_id': user_id,
                    'strategy_id': strategy_id,
                    'mode': mode,
                    'message': str(e),
                })

        return all_events

    def get_active_count(self) -> int:
        """Get the number of active strategy instances."""
        return len(self.active_instances)

    def get_active_for_user(self, user_id: int) -> list:
        """Get all active strategies for a specific user."""
        return [
            {'strategy_id': sid, 'mode': mode, 'instance': inst}
            for (uid, sid, mode), inst in self.active_instances.items()
            if uid == user_id
        ]

    def create_strategy_instance(
        self,
        mode: str,
        strategy_instance_id: str,
        strategy_id: str,
        strategy_name: str,
        user_id: int,
        generated_code: str,
        initial_balance: float = 10000.0,
        commission: float = 0.001,
        slippage: float = 0.0,
        price_provider: Any = None,
        exchange: Any = None,
        testnet: bool = True,
        max_position_pct: float = 0.5,
        circuit_breaker_pct: float = -0.10,
        indicator_owner_id: int = None,
    ) -> StrategyInstance:
        """
        Factory method to create the appropriate strategy instance based on mode.

        :param mode: Trading mode ('backtest', 'paper', 'live').
        :param strategy_instance_id: Unique instance identifier.
        :param strategy_id: Strategy identifier.
        :param strategy_name: Strategy name.
        :param user_id: User identifier.
        :param generated_code: Generated Python code from Blockly.
        :param initial_balance: Starting balance for paper/backtest.
        :param commission: Commission rate.
        :param slippage: Slippage rate.
        :param price_provider: Callable for getting current prices.
        :param exchange: Exchange instance for live trading.
        :param testnet: Use testnet for live trading (default True for safety).
        :param max_position_pct: Maximum position size as % of balance for live trading.
        :param circuit_breaker_pct: Drawdown % to halt trading for live trading.
        :param indicator_owner_id: For subscribed strategies, the creator's user ID for indicator lookup.
        :return: Strategy instance appropriate for the mode.
        """
        mode = mode.lower()

        if mode == TradingMode.PAPER:
            from paper_strategy_instance import PaperStrategyInstance
            return PaperStrategyInstance(
                strategy_instance_id=strategy_instance_id,
                strategy_id=strategy_id,
                strategy_name=strategy_name,
                user_id=user_id,
                generated_code=generated_code,
                data_cache=self.data_cache,
                indicators=self.indicators_manager,
                trades=self.trades,
                edm_client=self.edm_client,
                initial_balance=initial_balance,
                commission=commission,
                slippage=slippage if slippage > 0 else 0.0005,
                price_provider=price_provider,
                indicator_owner_id=indicator_owner_id,
            )

        elif mode == TradingMode.BACKTEST:
            from backtest_strategy_instance import BacktestStrategyInstance
            return BacktestStrategyInstance(
                strategy_instance_id=strategy_instance_id,
                strategy_id=strategy_id,
                strategy_name=strategy_name,
                user_id=user_id,
                generated_code=generated_code,
                data_cache=self.data_cache,
                indicators=self.indicators_manager,
                trades=self.trades,
                edm_client=self.edm_client,
                indicator_owner_id=indicator_owner_id,
            )

        elif mode == TradingMode.LIVE:
            if exchange is None:
                raise ValueError(
                    "Live trading requires an exchange instance. "
                    "Please configure exchange credentials first."
                )

            from live_strategy_instance import LiveStrategyInstance

            # Safety warning for production mode
            if not testnet:
                logger.warning(
                    f"Creating LiveStrategyInstance for PRODUCTION trading. "
                    f"Strategy: {strategy_name}, User: {user_id}"
                )
            else:
                logger.info(
                    f"Creating LiveStrategyInstance in TESTNET mode. "
                    f"Strategy: {strategy_name}, User: {user_id}"
                )

            return LiveStrategyInstance(
                strategy_instance_id=strategy_instance_id,
                strategy_id=strategy_id,
                strategy_name=strategy_name,
                user_id=user_id,
                generated_code=generated_code,
                data_cache=self.data_cache,
                indicators=self.indicators_manager,
                trades=self.trades,
                edm_client=self.edm_client,
                exchange=exchange,
                testnet=testnet,
                initial_balance=initial_balance,
                commission=commission,
                slippage=slippage,
                max_position_pct=max_position_pct,
                circuit_breaker_pct=circuit_breaker_pct,
                indicator_owner_id=indicator_owner_id,
            )

        else:
            # Default to standard StrategyInstance for unknown modes
            logger.warning(f"Unknown mode '{mode}'. Using standard StrategyInstance.")
            return StrategyInstance(
                strategy_instance_id=strategy_instance_id,
                strategy_id=strategy_id,
                strategy_name=strategy_name,
                user_id=user_id,
                generated_code=generated_code,
                data_cache=self.data_cache,
                indicators=self.indicators_manager,
                trades=self.trades,
                edm_client=self.edm_client,
                indicator_owner_id=indicator_owner_id,
            )

    def _save_strategy(self, strategy_data: dict, default_source: dict) -> dict:
        """
        Saves a strategy to the cache and database. Handles both creation and editing.

        :param strategy_data: A dictionary containing strategy data such as name, code, workspace, etc.
        :param default_source: The default source for undefined sources in the strategy.
        :return: A dictionary containing success or failure information.
        """
        is_edit = 'tbl_key' in strategy_data
        try:
            # Determine if this is an edit or a new creation
            tbl_key = strategy_data.get('tbl_key', str(uuid.uuid4()))

            if is_edit:
                # Verify the existing strategy
                existing_strategy = self.data_cache.get_rows_from_datacache(
                    cache_name='strategies',
                    filter_vals=[('tbl_key', tbl_key)],
                    include_tbl_key=True
                )
                if existing_strategy.empty:
                    return {"success": False, "message": "Strategy not found."}

                # Check if strategy is being made private (public -> private transition)
                was_public = bool(existing_strategy.iloc[0].get('public', 0))
                is_public = bool(strategy_data.get('public', 0))

                if was_public and not is_public:
                    # Strategy being made private - remove all subscriptions
                    self._remove_all_subscriptions_for_strategy(tbl_key)
                    logger.info(f"Strategy '{tbl_key}' made private - removed all subscriptions")

            else:
                # Check for duplicate strategy name
                filter_conditions = [
                    ('creator', strategy_data.get('creator')),
                    ('name', strategy_data['name'])
                ]
                existing_strategy = self.data_cache.get_rows_from_datacache(
                    cache_name='strategies',
                    filter_vals=filter_conditions,
                    include_tbl_key = True
                )
                if not existing_strategy.empty:
                    return {"success": False, "message": "A strategy with this name already exists"}

            # Check unique public name requirement
            is_public = bool(strategy_data.get('public', 0))
            if is_public:
                exclude_key = tbl_key if is_edit else None
                if not self._check_unique_public_name(strategy_data['name'], exclude_tbl_key=exclude_key):
                    return {"success": False, "message": "A public strategy with this name already exists"}

            # Validate and serialize 'workspace'
            workspace_data = strategy_data.get('workspace')
            if not isinstance(workspace_data, str) or not workspace_data.strip():
                return {"success": False, "message": "Invalid or empty workspace data"}

            # Serialize 'stats' field
            try:
                stats_data = strategy_data.get('stats', {})
                stats_serialized = json.dumps(stats_data)
            except (TypeError, ValueError):
                return {"success": False, "message": "Invalid stats data format"}

            # Validate and parse 'code'
            code = strategy_data.get('code')
            if isinstance(code, str):
                try:
                    strategy_json = json.loads(code)
                    if not isinstance(strategy_json, dict):
                        return {"success": False, "message": "'code' must be a JSON object."}
                except json.JSONDecodeError:
                    return {"success": False, "message": "Invalid JSON format for 'code'."}
            elif isinstance(code, dict):
                strategy_json = code
                strategy_data['code'] = json.dumps(strategy_json)  # Serialize for storage
            else:
                return {"success": False, "message": "'code' must be a JSON string or dictionary."}

            # Generate Python components using PythonGenerator
            python_generator = PythonGenerator(default_source.copy(), tbl_key)
            strategy_components = python_generator.generate(strategy_json)
            strategy_data['strategy_components'] = json.dumps(strategy_components)

            # Serialize default_source for storage
            default_source_serialized = json.dumps(default_source)

            # Prepare fields for database operations
            columns = (
                "creator", "name", "workspace", "code", "stats", "public", "fee", "tbl_key", "strategy_components", "default_source"
            )
            values = (
                strategy_data.get('creator'),
                strategy_data['name'],
                workspace_data,
                strategy_data['code'],
                stats_serialized,
                bool(strategy_data.get('public', 0)),
                float(strategy_data.get('fee', 0.0)),
                tbl_key,
                strategy_data['strategy_components'],
                default_source_serialized
            )

            if is_edit:
                # Update the existing strategy
                self.data_cache.modify_datacache_item(
                    cache_name='strategies',
                    filter_vals=[('tbl_key', tbl_key)],
                    field_names=columns,
                    new_values=values,
                    key=tbl_key,
                    overwrite='tbl_key'
                )
            else:
                # Insert a new strategy
                self.data_cache.insert_row_into_datacache(
                    cache_name='strategies',
                    columns=columns,
                    values=values
                )

            # Prepare the response
            response_strategy = strategy_data.copy()
            response_strategy.pop("strategy_components", None)  # Remove the sensitive field

            # Include `tbl_key` within the `strategy` object
            response_strategy['tbl_key'] = tbl_key

            # Include the default_source in the response for frontend display
            response_strategy['default_source'] = default_source

            return {
                "success": True,
                "message": "Strategy saved successfully",
                "strategy": response_strategy,
                "updated_at": dt.datetime.now(dt.timezone.utc).isoformat()
            }

        except Exception as e:
            # Handle exceptions and log errors
            logger.error(f"Failed to save strategy: {e}", exc_info=True)
            traceback.print_exc()
            operation = "update" if is_edit else "create"
            return {"success": False, "message": f"Failed to {operation} strategy: {str(e)}"}


    def new_strategy(self, strategy_data: dict, default_source: dict) -> dict:
        """
        Add a new strategy to the cache and database.

        :param default_source: The default source for undefined sources in the strategy.
        :param strategy_data: A dictionary containing strategy data such as name, code, and workspace.
        :return: A dictionary containing success or failure information.
        """
        return self._save_strategy(strategy_data, default_source)

    def edit_strategy(self, strategy_data: dict, default_source: dict) -> dict:
        """
        Updates an existing strategy in the cache and database.

        :param default_source: The default source for undefined sources in the strategy.
        :param strategy_data: A dictionary containing the updated strategy data.
        :return: A dictionary containing success or failure information.
        """
        return self._save_strategy(strategy_data, default_source)

    def verify_ownership(self, user_id: int, strategy_tbl_key: str) -> bool:
        """
        Verify that the user owns the strategy.

        :param user_id: The ID of the user to check.
        :param strategy_tbl_key: The tbl_key of the strategy.
        :return: True if the user owns the strategy, False otherwise.
        """
        if user_id is None or not strategy_tbl_key:
            return False

        strategy = self.get_strategy_by_tbl_key(strategy_tbl_key)
        if not strategy:
            return False

        creator = strategy.get('creator')
        if creator is None:
            return False

        try:
            return int(creator) == int(user_id)
        except (ValueError, TypeError):
            return False

    def _remove_all_subscriptions_for_strategy(self, strategy_tbl_key: str) -> None:
        """
        Remove all subscriptions to a strategy.

        Called when a strategy is deleted or made private.
        We explicitly delete subscriptions rather than relying on FK cascade
        since SQLite requires PRAGMA foreign_keys = ON.

        :param strategy_tbl_key: The tbl_key of the strategy.
        """
        try:
            # Remove from cache and database (remove_row_from_datacache handles both)
            self.data_cache.remove_row_from_datacache(
                cache_name='strategy_subscriptions',
                filter_vals=[('strategy_tbl_key', strategy_tbl_key)]
            )

            logger.info(f"Removed all subscriptions for strategy '{strategy_tbl_key}'")

        except Exception as e:
            logger.warning(f"Failed to remove subscriptions for strategy '{strategy_tbl_key}': {e}")

    def _get_username_for_id(self, user_id: int) -> str:
        """
        Get username for a user ID. Used for displaying creator names.

        :param user_id: The user's ID.
        :return: The username or a fallback string.
        """
        if user_id is None:
            return "Unknown"

        try:
            # Direct lookup via data_cache
            users_cache = self.data_cache.get_rows_from_datacache(
                cache_name='users',
                filter_vals=[('id', int(user_id))],
                include_tbl_key=True
            )
            if not users_cache.empty:
                return users_cache.iloc[0].get('user_name', f'User #{user_id}')
        except Exception as e:
            logger.debug(f"Failed to get username for user_id {user_id}: {e}")

        return f'User #{user_id}'  # Fallback

    def _redact_strategy_internals(self, strategy: dict) -> dict:
        """
        Remove sensitive internals from strategy for non-owners.

        This prevents subscribers from seeing the strategy's implementation details.

        :param strategy: The full strategy dictionary.
        :return: A redacted copy of the strategy.
        """
        redacted = strategy.copy()
        redacted['workspace'] = None
        redacted['code'] = None
        redacted['strategy_components'] = None  # CRITICAL: contains generated code
        return redacted

    def is_subscribed(self, user_id: int, strategy_tbl_key: str) -> bool:
        """
        Check if a user is subscribed to a strategy.

        :param user_id: The user's ID.
        :param strategy_tbl_key: The strategy's tbl_key.
        :return: True if subscribed, False otherwise.
        """
        if user_id is None or not strategy_tbl_key:
            return False

        try:
            subscriptions = self.data_cache.get_rows_from_datacache(
                cache_name='strategy_subscriptions',
                filter_vals=[('user_id', user_id), ('strategy_tbl_key', strategy_tbl_key)]
            )
            return not subscriptions.empty
        except Exception:
            return False

    def _is_strategy_running_for_user(self, user_id: int, strategy_tbl_key: str) -> bool:
        """
        Check if user has a running instance of the given strategy.

        :param user_id: The user's ID.
        :param strategy_tbl_key: The strategy's tbl_key.
        :return: True if the strategy is running for this user.
        """
        for (uid, sid, mode) in self.active_instances.keys():
            if uid == user_id and sid == strategy_tbl_key:
                return True
        return False

    def subscribe_to_strategy(self, user_id: int, strategy_tbl_key: str) -> dict:
        """
        Subscribe a user to a public strategy.

        :param user_id: The user's ID.
        :param strategy_tbl_key: The strategy's tbl_key.
        :return: A dictionary indicating success or failure.
        """
        if user_id is None or not strategy_tbl_key:
            return {"success": False, "message": "Invalid user or strategy"}

        # Get the strategy
        strategy = self.get_strategy_by_tbl_key(strategy_tbl_key)
        if not strategy:
            return {"success": False, "message": "Strategy not found"}

        # Check if strategy is public
        if not strategy.get('public'):
            return {"success": False, "message": "Cannot subscribe to private strategy"}

        # Check if user is the owner
        if self.verify_ownership(user_id, strategy_tbl_key):
            return {"success": False, "message": "You cannot subscribe to your own strategy"}

        # Check if already subscribed
        if self.is_subscribed(user_id, strategy_tbl_key):
            return {"success": False, "message": "Already subscribed to this strategy"}

        try:
            # Add subscription via datacache (handles both cache and DB insert)
            subscribed_at = dt.datetime.now(dt.timezone.utc).isoformat()
            self.data_cache.insert_row_into_datacache(
                cache_name='strategy_subscriptions',
                columns=("user_id", "strategy_tbl_key", "subscribed_at"),
                values=(user_id, strategy_tbl_key, subscribed_at)
            )

            logger.info(f"User {user_id} subscribed to strategy '{strategy_tbl_key}'")
            return {
                "success": True,
                "message": "Successfully subscribed to strategy",
                "strategy_name": strategy.get('name')
            }

        except Exception as e:
            logger.error(f"Failed to subscribe user {user_id} to strategy '{strategy_tbl_key}': {e}")
            return {"success": False, "message": f"Failed to subscribe: {str(e)}"}

    def unsubscribe_from_strategy(self, user_id: int, strategy_tbl_key: str) -> dict:
        """
        Unsubscribe a user from a strategy.

        :param user_id: The user's ID.
        :param strategy_tbl_key: The strategy's tbl_key.
        :return: A dictionary indicating success or failure.
        """
        if user_id is None or not strategy_tbl_key:
            return {"success": False, "message": "Invalid user or strategy"}

        # Check if subscribed
        if not self.is_subscribed(user_id, strategy_tbl_key):
            return {"success": False, "message": "Not subscribed to this strategy"}

        # Check if strategy is running for this user
        if self._is_strategy_running_for_user(user_id, strategy_tbl_key):
            return {"success": False, "message": "Stop the strategy before unsubscribing"}

        try:
            # Remove from cache and database (remove_row_from_datacache handles both)
            self.data_cache.remove_row_from_datacache(
                cache_name='strategy_subscriptions',
                filter_vals=[('user_id', user_id), ('strategy_tbl_key', strategy_tbl_key)]
            )

            logger.info(f"User {user_id} unsubscribed from strategy '{strategy_tbl_key}'")
            return {"success": True, "message": "Successfully unsubscribed from strategy"}

        except Exception as e:
            logger.error(f"Failed to unsubscribe user {user_id} from strategy '{strategy_tbl_key}': {e}")
            return {"success": False, "message": f"Failed to unsubscribe: {str(e)}"}

    def get_user_strategies(self, user_id: int) -> list:
        """
        Get strategies that a user owns OR is subscribed to.

        This is the primary method for getting a user's strategy list.
        Subscribed strategies are redacted (no code/workspace visible).

        :param user_id: The user's ID.
        :return: List of strategy dictionaries.
        """
        result = []

        if user_id is None:
            return result

        # Get user's own strategies (public or private)
        owned = self.data_cache.get_rows_from_datacache(
            cache_name='strategies',
            filter_vals=[('creator', user_id)],
            include_tbl_key=True
        )

        # Add owned strategies (full access)
        if owned is not None and not owned.empty:
            for _, row in owned.iterrows():
                strat = row.to_dict()
                strat['is_owner'] = True
                strat['is_subscribed'] = False
                # Deserialize JSON fields
                if isinstance(strat.get('default_source'), str):
                    try:
                        strat['default_source'] = json.loads(strat['default_source'])
                    except json.JSONDecodeError:
                        strat['default_source'] = {}
                if isinstance(strat.get('stats'), str):
                    try:
                        strat['stats'] = json.loads(strat['stats'])
                    except json.JSONDecodeError:
                        strat['stats'] = {}
                result.append(strat)

        # Get subscribed strategy keys
        subscriptions = self.data_cache.get_rows_from_datacache(
            cache_name='strategy_subscriptions',
            filter_vals=[('user_id', user_id)]
        )

        # Add subscribed strategies (redacted)
        if subscriptions is not None and not subscriptions.empty:
            for _, sub in subscriptions.iterrows():
                strategy_key = sub.get('strategy_tbl_key')
                strategy = self.get_strategy_by_tbl_key(strategy_key)

                if strategy and strategy.get('public'):  # Still public
                    strat = self._redact_strategy_internals(strategy)
                    strat['is_owner'] = False
                    strat['is_subscribed'] = True
                    strat['creator_name'] = self._get_username_for_id(strategy.get('creator'))
                    result.append(strat)

        return result

    def get_public_strategies_catalog(self, user_id: int) -> list:
        """
        Get all public strategies for the browse dialog (sanitized).

        Does not include the user's own strategies.

        :param user_id: The user's ID (to exclude their own strategies).
        :return: List of sanitized strategy dictionaries.
        """
        result = []

        # Get all public strategies
        public = self.data_cache.get_rows_from_datacache(
            cache_name='strategies',
            filter_vals=[('public', 1)],
            include_tbl_key=True
        )

        if public is None or public.empty:
            return result

        for _, row in public.iterrows():
            creator = row.get('creator')
            tbl_key = row.get('tbl_key')

            # Skip user's own strategies
            try:
                if user_id is not None and int(creator) == int(user_id):
                    continue
            except (ValueError, TypeError):
                pass

            strat = self._redact_strategy_internals(row.to_dict())
            strat['creator_name'] = self._get_username_for_id(creator)
            strat['is_subscribed'] = self.is_subscribed(user_id, tbl_key)
            result.append(strat)

        return result

    def _check_unique_public_name(self, name: str, exclude_tbl_key: str = None) -> bool:
        """
        Check if a public strategy name is unique.

        :param name: The strategy name to check.
        :param exclude_tbl_key: Optional tbl_key to exclude (for edit operations).
        :return: True if the name is unique among public strategies.
        """
        public_with_name = self.data_cache.get_rows_from_datacache(
            cache_name='strategies',
            filter_vals=[('public', 1), ('name', name)],
            include_tbl_key=True
        )

        if public_with_name is None or public_with_name.empty:
            return True  # Name is unique

        # Check all matches, not just first
        for _, row in public_with_name.iterrows():
            if row.get('tbl_key') != exclude_tbl_key:
                return False  # Found another public strategy with same name

        return True

    def delete_strategy(self, tbl_key: str, user_id: int = None) -> dict:
        """
        Deletes a strategy identified by its tbl_key.

        Security: If user_id is provided, ownership is verified before deletion.
        Also removes all subscriptions to this strategy (explicit cleanup, not FK cascade).

        :param tbl_key: The unique identifier of the strategy to delete.
        :param user_id: The ID of the user requesting deletion (for ownership check).
        :return: A dictionary indicating success or failure with an appropriate message.
        """
        # Ownership check if user_id is provided
        if user_id is not None:
            if not self.verify_ownership(user_id, tbl_key):
                logger.warning(f"User {user_id} attempted to delete strategy '{tbl_key}' without ownership")
                return {"success": False, "message": "You don't own this strategy."}

        try:
            # Remove all subscriptions first (explicit cleanup, don't rely on FK cascade)
            self._remove_all_subscriptions_for_strategy(tbl_key)

            # Then delete the strategy
            self.data_cache.remove_row_from_datacache(
                cache_name='strategies',
                filter_vals=[('tbl_key', tbl_key)]
            )
            return {"success": True, "message": "Strategy deleted successfully."}
        except Exception as e:
            logger.error(f"Failed to delete strategy '{tbl_key}': {e}", exc_info=True)
            return {"success": False, "message": f"Failed to delete strategy: {str(e)}"}

    def cleanup_orphaned_strategies(self) -> dict:
        """
        Removes strategies that belong to non-existent users from the database.
        This is a maintenance operation to clean up orphaned data.

        :return: A dictionary with cleanup results.
        """
        try:
            # Get all strategies without filtering
            all_strategies = self.data_cache.get_all_rows_from_datacache(cache_name='strategies')

            if all_strategies is None or all_strategies.empty:
                return {"success": True, "removed": 0, "message": "No strategies found."}

            removed_count = 0
            removed_names = []

            for idx, row in all_strategies.iterrows():
                creator = row.get('creator')
                tbl_key = row.get('tbl_key')
                name = row.get('name', 'unknown')

                # Check for invalid or non-existent creator
                should_remove = False
                reason = ""

                if creator is None:
                    should_remove = True
                    reason = "null creator"
                elif isinstance(creator, bytes):
                    should_remove = True
                    reason = "corrupted creator (bytes)"
                elif isinstance(creator, str) and not creator.isdigit():
                    should_remove = True
                    reason = f"invalid creator format: {creator}"
                else:
                    try:
                        creator_id = int(creator)
                        if not self._user_exists(creator_id):
                            should_remove = True
                            reason = f"user {creator_id} does not exist"
                    except (ValueError, TypeError):
                        should_remove = True
                        reason = "creator conversion error"

                if should_remove and tbl_key:
                    logger.info(f"Removing orphaned strategy '{name}' (tbl_key: {tbl_key}) - {reason}")
                    self.delete_strategy(tbl_key)
                    removed_count += 1
                    removed_names.append(name)

            return {
                "success": True,
                "removed": removed_count,
                "removed_strategies": removed_names,
                "message": f"Cleaned up {removed_count} orphaned strategies."
            }

        except Exception as e:
            logger.error(f"Failed to cleanup orphaned strategies: {e}", exc_info=True)
            return {"success": False, "removed": 0, "message": f"Cleanup failed: {str(e)}"}

    def get_all_strategy_names(self, user_id: int) -> list | None:
        """
        Return a list of all public and user strategy names stored in the cache or database.
        """
        # Fetch all strategy names from the cache or database
        strategies_df = self.get_all_strategies(user_id, 'df')

        if not strategies_df.empty:
            return strategies_df['name'].tolist()
        return None

    def _user_exists(self, user_id: int) -> bool:
        """
        Check if a user exists in the users cache/database.

        :param user_id: The user ID to check.
        :return: True if the user exists, False otherwise.
        """
        if user_id is None:
            return False
        try:
            # Try to get the username for this user_id
            username = self.data_cache.get_datacache_item(
                item_name='user_name',
                cache_name='users',
                filter_vals=('id', int(user_id))
            )
            return username is not None
        except (ValueError, TypeError):
            # Invalid user_id format (e.g., corrupted data)
            return False

    def _filter_valid_strategies(self, strategies_df: pd.DataFrame) -> pd.DataFrame:
        """
        Filter out strategies with non-existent or invalid creators.

        :param strategies_df: DataFrame of strategies to filter.
        :return: Filtered DataFrame with only valid strategies.
        """
        if strategies_df is None or strategies_df.empty:
            return strategies_df

        valid_indices = []
        for idx, row in strategies_df.iterrows():
            creator = row.get('creator')
            # Check for valid creator
            try:
                # Handle corrupted data (binary garbage, etc.)
                if creator is None:
                    continue
                if isinstance(creator, bytes):
                    logger.warning(f"Skipping strategy with corrupted creator field (bytes): {row.get('name', 'unknown')}")
                    continue
                if isinstance(creator, str) and not creator.isdigit():
                    logger.warning(f"Skipping strategy with invalid creator field: {row.get('name', 'unknown')}")
                    continue

                creator_id = int(creator)
                if self._user_exists(creator_id):
                    valid_indices.append(idx)
                else:
                    logger.debug(f"Filtering out strategy '{row.get('name', 'unknown')}' - creator {creator_id} does not exist")
            except (ValueError, TypeError) as e:
                logger.warning(f"Skipping strategy with invalid creator: {row.get('name', 'unknown')} - {e}")
                continue

        return strategies_df.loc[valid_indices].reset_index(drop=True)

    def get_all_strategies(self, user_id: int | None, form: str, include_all: bool = False):
        """
        Return stored strategies in various formats.

        :param user_id: The ID of the user making the request. If None, fetch all public strategies.
        :param form: The desired format ('obj', 'json', or 'dict').
        :param include_all: If True, fetch all strategies (public and private) for all users.
        :return: A list of strategies in the requested format.
        """
        valid_forms = {'df', 'json', 'dict'}
        if form not in valid_forms:
            raise ValueError(f"Invalid form '{form}'. Expected one of {valid_forms}.")

        if include_all:
            # Fetch all strategies regardless of user or public status
            strategies_df = self.data_cache.get_all_rows_from_datacache(
                cache_name='strategies',
            )
        else:
            # Fetch public strategies
            public_df = self.data_cache.get_rows_from_datacache(
                cache_name='strategies',
                filter_vals=[('public', 1)],
                include_tbl_key=True
            )

            if user_id is not None:
                # Fetch user-specific private strategies
                user_df = self.data_cache.get_rows_from_datacache(
                    cache_name='strategies',
                    filter_vals=[('creator', user_id), ('public', 0)],
                    include_tbl_key=True
                )
                # Concatenate the two DataFrames
                strategies_df = pd.concat([public_df, user_df], ignore_index=True)
            else:
                strategies_df = public_df

        # Filter out strategies from non-existent or invalid users
        strategies_df = self._filter_valid_strategies(strategies_df)

        # Return None if no strategies found
        if strategies_df is None or strategies_df.empty:
            return None

        # Return the strategies in the requested format
        if form == 'df':
            return strategies_df
        elif form == 'json':
            return strategies_df.to_json(orient='records')
        elif form == 'dict':
            records = strategies_df.to_dict('records')
            # Deserialize JSON string fields for each record
            for record in records:
                # Deserialize 'default_source' if stored as JSON string
                if isinstance(record.get('default_source'), str):
                    try:
                        record['default_source'] = json.loads(record['default_source'])
                    except json.JSONDecodeError:
                        record['default_source'] = {}
                # Deserialize 'stats' if stored as JSON string
                if isinstance(record.get('stats'), str):
                    try:
                        record['stats'] = json.loads(record['stats'])
                    except json.JSONDecodeError:
                        record['stats'] = {}
            return records

    def get_strategy_by_tbl_key(self, tbl_key: str) -> dict[str, Any] | None:
        """
        Retrieve a strategy object by tbl_key.

        :param tbl_key: The unique identifier of the strategy.
        :return: The strategy dictionary if found, else None.
        """
        strategies_df = self.get_all_strategies(None, 'df', include_all=True)

        if strategies_df is None:
            return None

        filtered_strategies = strategies_df.query('tbl_key == @tbl_key')

        if filtered_strategies.empty:
            return None

        strategy_row = filtered_strategies.iloc[0].to_dict()

        # Deserialize the 'strategy_components' field
        try:
            strategy_components = json.loads(strategy_row.get('strategy_components', '{}'))
        except json.JSONDecodeError:
            strategy_components = {}
        strategy_row['strategy_components'] = strategy_components

        # If 'code' is stored as a JSON string, deserialize it
        if isinstance(strategy_row.get('code'), str):
            strategy_row['code'] = json.loads(strategy_row['code'])

        # Deserialize 'default_source' if stored as JSON string
        if isinstance(strategy_row.get('default_source'), str):
            try:
                strategy_row['default_source'] = json.loads(strategy_row['default_source'])
            except json.JSONDecodeError:
                strategy_row['default_source'] = {}

        return strategy_row

    def get_required_exchanges(self, strategy_tbl_key: str) -> set[str]:
        """
        Get the set of exchange names required by a strategy.

        Extracts unique exchange names from the strategy's data sources
        and default source settings.

        :param strategy_tbl_key: The unique identifier of the strategy.
        :return: Set of canonicalized exchange names required by the strategy.
        """
        from exchange_validation import extract_required_exchanges
        strategy = self.get_strategy_by_tbl_key(strategy_tbl_key)
        return extract_required_exchanges(strategy)

    def update_strategy_stats(self, strategy_id: str, profit_loss: float) -> None:
        """
        Updates the strategy's statistics based on the latest profit or loss.

        :param strategy_id: Identifier of the strategy.
        :param profit_loss: Latest profit or loss from strategy execution.
        """
        try:
            # Fetch the current stats
            strategy = self.data_cache.get_rows_from_datacache(cache_name='strategies',
                                                               filter_vals=[('tbl_key', strategy_id)],
                                                               include_tbl_key=True)

            if strategy.empty:
                logger.warning(f"Strategy ID {strategy_id} not found for stats update.")
                return

            strategy_row = strategy.iloc[0].to_dict()
            current_stats = json.loads(strategy_row.get('stats', '{}'))

            # Update stats
            current_stats['num_trades'] = current_stats.get('num_trades', 0) + 1
            current_stats['total_position'] = current_stats.get('total_position', 0.0) + profit_loss
            current_stats['total_position_value'] = current_stats.get('total_position_value', 0.0) + profit_loss

            # Serialize and update in datacache
            self.data_cache.modify_datacache_item(cache_name='strategies', filter_vals=[('tbl_key', strategy_id)],
                                                  field_names=('stats',), new_values=(json.dumps(current_stats),),
                                                  key=strategy_id, overwrite='tbl_key')
            logger.info(f"Updated stats for strategy {strategy_id}: {current_stats}")

        except Exception as e:
            logger.error(f"Error updating stats for strategy '{strategy_id}': {e}", exc_info=True)

    def execute_strategy(
        self,
        strategy_data: dict[str, Any],
        mode: str = 'backtest',
        initial_balance: float = 10000.0,
        commission: float = 0.001,
        slippage: float = 0.0,
        price_provider: Any = None,
        exchange: Any = None,
        testnet: bool = True,
        max_position_pct: float = 0.5,
        circuit_breaker_pct: float = -0.10,
    ) -> dict[str, Any]:
        """
        Executes a strategy based on the provided strategy data.

        :param strategy_data: A dictionary containing strategy details.
        :param mode: Trading mode ('backtest', 'paper', 'live').
        :param initial_balance: Starting balance for paper/backtest modes.
        :param commission: Commission rate.
        :param slippage: Slippage rate for market orders.
        :param price_provider: Callable for getting current prices (paper/live).
        :param exchange: Exchange instance for live trading.
        :param testnet: Use testnet for live trading (default True for safety).
        :param max_position_pct: Maximum position size as % of balance for live trading.
        :param circuit_breaker_pct: Drawdown % to halt trading for live trading.
        :return: A dictionary indicating success or failure with relevant messages.
        """
        try:
            # Extract identifiers
            strategy_id = strategy_data.get('tbl_key')
            strategy_name = strategy_data.get('name')
            user_id = strategy_data.get('creator')

            if not strategy_id or not strategy_name or not user_id:
                return {"success": False, "message": "Strategy data is incomplete."}

            # Generate a deterministic strategy_instance_id (include mode for uniqueness)
            strategy_instance_id = f"{user_id}_{strategy_name}_{mode}"
            instance_key = (user_id, strategy_id, mode)  # Include mode in key

            # Retrieve or create StrategyInstance
            if instance_key not in self.active_instances:
                generated_code = strategy_data.get('strategy_components', {}).get('generated_code', '')
                if not generated_code:
                    return {"success": False, "message": "No 'next()' method defined for the strategy."}

                # Use factory method to create appropriate instance for mode
                strategy_instance = self.create_strategy_instance(
                    mode=mode,
                    strategy_instance_id=strategy_instance_id,
                    strategy_id=strategy_id,
                    strategy_name=strategy_name,
                    user_id=user_id,
                    generated_code=generated_code,
                    initial_balance=initial_balance,
                    commission=commission,
                    slippage=slippage,
                    price_provider=price_provider,
                    exchange=exchange,
                    testnet=testnet,
                    max_position_pct=max_position_pct,
                    circuit_breaker_pct=circuit_breaker_pct,
                )

                # Store in active_instances
                self.active_instances[instance_key] = strategy_instance
                logger.debug(f"Created new StrategyInstance '{strategy_instance_id}' for strategy '{strategy_id}'.")

            else:
                strategy_instance = self.active_instances[instance_key]
                logger.debug(
                    f"Retrieved existing StrategyInstance '{strategy_instance_id}' for strategy '{strategy_id}'.")

            # Execute the strategy
            execution_result = strategy_instance.execute()

            if execution_result.get('success'):
                profit_loss = execution_result.get('profit_loss', 0.0)
                self.update_strategy_stats(strategy_id, profit_loss)
                logger.info(f"Strategy '{strategy_id}' executed successfully with profit/loss: {profit_loss}")

                # Check if exit was initiated
                if strategy_instance.exit:
                    # Check if all trades are closed
                    if self.trades.all_trades_closed(strategy_id):
                        logger.info(f"Strategy '{strategy_id}' has successfully exited all trades.")
                        # Remove from active_instances
                        del self.active_instances[instance_key]
                        # Optionally, mark the strategy as inactive in the database
                        self.data_cache.modify_datacache_item(
                            cache_name='strategies',
                            filter_vals=[('tbl_key', strategy_id)],
                            field_names=('active',),
                            new_values=(False,),
                            key=strategy_id,
                            overwrite='tbl_key'
                        )
                    else:
                        logger.info(
                            f"Strategy '{strategy_id}' is exiting. Remaining trades will be closed in subsequent executions.")

                return {"success": True, "strategy_profit_loss": profit_loss}
            else:
                error_message = execution_result.get('message', 'Unknown error.')
                logger.error(f"Strategy '{strategy_id}' execution failed: {error_message}")
                return {"success": False, "message": f"Strategy execution failed: {error_message}"}

        except Exception as e:
            logger.error(f"Unexpected error in execute_strategy: {e}")
            traceback.print_exc()
            return {"success": False, "message": f"Unexpected error: {str(e)}"}

    def update_db_active_strategies(self):
        """
        Legacy method: Loops through and executes all DB-flagged active strategies.

        Note: This is different from update() which handles the execution loop
        for strategies started via run_strategy. This method reads from the
        'active' flag in the database, which is a different activation mechanism.
        """
        try:
            active_strategies = self.data_cache.get_rows_from_datacache('strategies',
                                                                        [('active', True)],
                                                                        include_tbl_key=True)
            if active_strategies.empty:
                logger.info("No DB-active strategies to execute.")
                return  # No active strategies to execute
            for _, strategy_data in active_strategies.iterrows():
                self.execute_strategy(strategy_data)
        except Exception as e:
            logger.error(f"Error updating DB-active strategies: {e}", exc_info=True)
            traceback.print_exc()

    def update_stats(self, tbl_key: str, stats: dict) -> None:
        """
        Updates the strategy's statistics with the provided stats.

        :param tbl_key: Identifier of the strategy (tbl_key).
        :param stats: Dictionary containing statistics to update.
        """
        try:
            # Fetch the current strategy data
            strategy = self.data_cache.get_rows_from_datacache(
                cache_name='strategies',
                filter_vals=[('tbl_key', tbl_key)],
                include_tbl_key=True)

            if strategy.empty:
                logger.warning(f"Strategy ID {tbl_key} not found for stats update.")
                return

            strategy_row = strategy.iloc[0].to_dict()
            current_stats = json.loads(strategy_row.get('stats', '{}'))

            # Merge the new stats with existing stats
            current_stats.update(stats)

            # Serialize the updated stats
            updated_stats_serialized = json.dumps(current_stats)

            # Update the stats in the data cache
            self.data_cache.modify_datacache_item(
                cache_name='strategies',
                filter_vals=[('tbl_key', tbl_key)],
                field_names=('stats',),
                new_values=(updated_stats_serialized,),
                key=tbl_key,
                overwrite='tbl_key'
            )

            logger.info(f"Updated stats for strategy '{tbl_key}': {current_stats}")

        except Exception as e:
            logger.error(f"Error updating stats for strategy '{tbl_key}': {e}", exc_info=True)
