Refactored DataCache, again. Implemented more advance cache management. All DataCache tests pass.

src/BrighterTrades.py

@@ -1,7 +1,7 @@
 from typing import Any
 
 from Users import Users
-from DataCache_v2 import DataCache
+from DataCache_v3 import DataCache
 from Strategies import Strategies
 from backtesting import Backtester
 from candles import Candles
@@ -503,8 +503,6 @@ class BrighterTrades:
             print(f'ERROR SETTING VALUE')
             print(f'The string received by the server was: /n{params}')
 
-        # Save any changes to storage
-        self.config.config_and_states('save')
         # Now that the state is changed reload price history.
         self.candles.set_cache(user_name=user_name)
         return

src/DataCache_v2.py

@@ -94,10 +94,35 @@ class DataCache:
         self.db = Database()
         self.exchanges = exchanges
         # Single DataFrame for all cached data
-        self.cache = pd.DataFrame(columns=['key', 'data'])  # Assuming 'key' and 'data' are necessary
+        self.caches = {}
         logger.info("DataCache initialized.")
 
-    def fetch_cached_rows(self, table: str, filter_vals: Tuple[str, Any]) -> pd.DataFrame | None:
+    def set_cache(self, data: Any, key: str, do_not_overwrite: bool = False) -> None:
+        """
+        Sets or updates an entry in the cache with the provided key. If the key already exists, the existing entry
+        is replaced unless `do_not_overwrite` is True. In that case, the existing entry is preserved.
+
+        Parameters:
+        data: The data to be cached. This can be of any type.
+        key: The unique key used to identify the cached data.
+        do_not_overwrite : The default is False, meaning that the existing entry will be replaced.
+        """
+        if do_not_overwrite and key in self.cache['key'].values:
+            return
+
+        # Construct a new DataFrame row with the key and data
+        new_row = pd.DataFrame({'key': [key], 'data': [data]})
+
+        # If the key already exists in the cache, remove the old entry
+        self.cache = self.cache[self.cache['key'] != key]
+
+        # Append the new row to the cache
+        self.cache = pd.concat([self.cache, new_row], ignore_index=True)
+
+        print(f'Current Cache: {self.cache}')
+        logger.debug(f'Cache set for key: {key}')
+
+    def get_or_fetch_rows(self, table: str, filter_vals: Tuple[str, Any]) -> pd.DataFrame | None:
         """
         Retrieves rows from the cache if available; otherwise, queries the database and caches the result.
 
@@ -170,10 +195,10 @@ class DataCache:
         :return: True if the attribute is already taken, False otherwise.
         """
         # Fetch rows from the specified table where the attribute matches the given value
-        result = self.fetch_cached_rows(table=table, filter_vals=(attr, val))
+        result = self.get_or_fetch_rows(table=table, filter_vals=(attr, val))
         return result is not None and not result.empty
 
-    def fetch_cached_item(self, item_name: str, table_name: str, filter_vals: Tuple[str, Any]) -> Any:
+    def fetch_item(self, item_name: str, table_name: str, filter_vals: Tuple[str, Any]) -> Any:
         """
         Retrieves a specific item from the cache or database, caching the result if necessary.
 
@@ -183,16 +208,16 @@ class DataCache:
         :return: The value of the requested item.
         :raises ValueError: If the item is not found in either the cache or the database.
         """
-        # Fetch the relevant rows
+        # Fetch the relevant rows from the cache or database
-        rows = self.fetch_cached_rows(table_name, filter_vals)
+        rows = self.get_or_fetch_rows(table_name, filter_vals)
         if rows is not None and not rows.empty:
             # Return the specific item from the first matching row.
             return rows.iloc[0][item_name]
 
         # If the item is not found, raise an error.
-        raise ValueError(f"Item {item_name} not found in {table_name} where {filter_vals[0]} = {filter_vals[1]}")
+        raise ValueError(f"Item '{item_name}' not found in '{table_name}' where {filter_vals[0]} = {filter_vals[1]}")
 
-    def modify_cached_row(self, table: str, filter_vals: Tuple[str, Any], field_name: str, new_data: Any) -> None:
+    def modify_item(self, table: str, filter_vals: Tuple[str, Any], field_name: str, new_data: Any) -> None:
         """
         Modifies a specific field in a row within the cache and updates the database accordingly.
 
@@ -202,7 +227,7 @@ class DataCache:
         :param new_data: The new data to be set.
         """
         # Retrieve the row from the cache or database
-        row = self.fetch_cached_rows(table, filter_vals)
+        row = self.get_or_fetch_rows(table, filter_vals)
 
         if row is None or row.empty:
             raise ValueError(f"Row not found in cache or database for {filter_vals[0]} = {filter_vals[1]}")
@@ -223,7 +248,7 @@ class DataCache:
         # Update the database with the modified row
         self.db.insert_dataframe(row.drop(columns='id'), table)
 
-    def insert_data(self, df: pd.DataFrame, table: str, skip_cache: bool = False) -> None:
+    def insert_df(self, df: pd.DataFrame, table: str, skip_cache: bool = False) -> None:
         """
         Inserts data into the specified table in the database, with an option to skip cache insertion.
 
@@ -569,22 +594,6 @@ class DataCache:
         else:
             raise KeyError(f"Cache key '{cache_key}' not found.")
 
-    def set_cache(self, data: Any, key: str, do_not_overwrite: bool = False) -> None:
-        if do_not_overwrite and key in self.cache['key'].values:
-            return
-
-        # Corrected construction of the new row
-        new_row = pd.DataFrame({'key': [key], 'data': [data]})
-
-        # If the key already exists, drop the old entry
-        self.cache = self.cache[self.cache['key'] != key]
-
-        # Append the new row to the cache
-        self.cache = pd.concat([self.cache, new_row], ignore_index=True)
-
-        print(f'Current Cache: {self.cache}')
-        logger.debug(f'Cache set for key: {key}')
-
     def _fetch_candles_from_exchange(self, symbol: str, interval: str, exchange_name: str, user_name: str,
                                      start_datetime: dt.datetime = None,
                                      end_datetime: dt.datetime = None) -> pd.DataFrame:

src/Database.py

@@ -133,29 +133,41 @@ class Database:
             print(f"Error querying table '{table}' for column '{filter_vals[0]}': {e}")
             return None
 
-    def insert_dataframe(self, df: pd.DataFrame, table: str) -> None:
+    def insert_dataframe(self, df: pd.DataFrame, table: str) -> int:
         """
-        Inserts a DataFrame into a specified table.
+        Inserts a DataFrame into a specified table and returns the last inserted row's ID.
 
         :param df: DataFrame to insert.
         :param table: Name of the table.
+        :return: The auto-incremented ID of the last inserted row.
         """
         with SQLite(self.db_file) as con:
+            # Insert the DataFrame into the specified table
             df.to_sql(name=table, con=con, index=False, if_exists='append')
 
-    def insert_row(self, table: str, columns: Tuple[str, ...], values: Tuple[Any, ...]) -> None:
+            # Fetch the last inserted row ID
+            cursor = con.execute('SELECT last_insert_rowid()')
+            last_id = cursor.fetchone()[0]
+
+        return last_id
+
+    def insert_row(self, table: str, columns: Tuple[str, ...], values: Tuple[Any, ...]) -> int:
         """
-        Inserts a row into a specified table.
+        Inserts a row into a specified table and returns the auto-incremented ID.
 
         :param table: Name of the table.
         :param columns: Tuple of column names.
         :param values: Tuple of values to insert.
+        :return: The auto-incremented ID of the inserted row.
         """
         with SQLite(self.db_file) as conn:
             cursor = conn.cursor()
             sql = make_insert(table=table, columns=columns)
             cursor.execute(sql, values)
 
+            # Return the auto-incremented ID
+            return cursor.lastrowid
+
     def table_exists(self, table_name: str) -> bool:
         """
         Checks if a table exists in the database.

src/Strategies.py

@@ -1,12 +1,26 @@
 import json
 from DataCache_v2 import DataCache
 
+
 class Strategy:
     def __init__(self, **args):
         """
         :param args: An object containing key_value pairs representing strategy attributes.
                     Strategy format is defined in strategies.js
         """
+        self.active = None
+        self.type = None
+        self.trade_amount = None
+        self.max_position = None
+        self.side = None
+        self.trd_in_conds = None
+        self.merged_loss = None
+        self.gross_loss = None
+        self.stop_loss = None
+        self.take_profit = None
+        self.gross_profit = None
+        self.merged_profit = None
+        self.name = None
         self.current_value = None
         self.opening_value = None
         self.gross_pl = None
@@ -161,19 +175,22 @@ class Strategies:
         # Reference to the trades object that maintains all trading actions and data.
         self.trades = trades
 
+        self.strat_list = []
+
     def get_all_strategy_names(self) -> list | None:
         """Return a list of all strategies in the database"""
-        self.data._get_from_database()
+        # # Load existing Strategies from file.
-        # Load existing Strategies from file.
+        # loaded_strategies = self.data.get_setting('strategies')
-        loaded_strategies = config.get_setting('strategies')
+        # if loaded_strategies is None:
-        if loaded_strategies is None:
+        #     # Populate the list and file with defaults defined in this class.
-            # Populate the list and file with defaults defined in this class.
+        #     loaded_strategies = self.get_strategy_defaults()
-            loaded_strategies = self.get_strategy_defaults()
+        #     config.set_setting('strategies', loaded_strategies)
-            config.set_setting('strategies', loaded_strategies)
+        # 
+        # for entry in loaded_strategies:
+        #     # Initialise all the strategy objects with data from file.
+        #     self.strat_list.append(Strategy(**entry))
 
-        for entry in loaded_strategies:
+        return None
-            # Initialise all the strategy objects with data from file.
-            self.strat_list.append(Strategy(**entry))        return None
 
     def new_strategy(self, data):
         # Create an instance of the new Strategy.

src/Users.py

@@ -4,7 +4,7 @@ import random
 from typing import Any
 from passlib.hash import bcrypt
 import pandas as pd
-from DataCache_v2 import DataCache
+from DataCache_v3 import DataCache
 
 
 class BaseUser:
@@ -28,9 +28,9 @@ class BaseUser:
         :param user_name: The name of the user.
         :return: The ID of the user as an integer.
         """
-        return self.data.fetch_cached_item(
+        return self.data.fetch_item(
             item_name='id',
-            table_name='users',
+            cache_name='users',
             filter_vals=('user_name', user_name)
         )
 
@@ -40,10 +40,10 @@ class BaseUser:
 
         :param user_name: The name of the user to remove from the cache.
         """
-        # Use DataCache to remove the user from the cache only
+        # Remove the user from the cache only
         self.data.remove_row(
-            table='users',
+            cache_name='users',
-            filter_vals=('user_name', user_name)
+            filter_vals=('user_name', user_name), remove_from_db=False
         )
 
     def delete_user(self, user_name: str) -> None:
@@ -54,7 +54,7 @@ class BaseUser:
         """
         self.data.remove_row(
             filter_vals=('user_name', user_name),
-            table='users'
+            cache_name='users'
         )
 
     def get_user_data(self, user_name: str) -> pd.DataFrame | None:
@@ -67,8 +67,8 @@ class BaseUser:
         :raises ValueError: If the user is not found in both the cache and the database.
         """
         # Attempt to fetch the user data from the cache or database via DataCache
-        user = self.data.fetch_cached_rows(
+        user = self.data.get_or_fetch_rows(
-            table='users',
+            cache_name='users',
             filter_vals=('user_name', user_name)
         )
 
@@ -86,8 +86,8 @@ class BaseUser:
         :param new_data: The new data to be set.
         """
         # Use DataCache to modify the user's data
-        self.data.modify_cached_row(
+        self.data.modify_item(
-            table='users',
+            cache_name='users',
             filter_vals=('user_name', username),
             field_name=field_name,
             new_data=new_data
@@ -112,8 +112,8 @@ class UserAccountManagement(BaseUser):
         self.max_guests = max_guests  # Maximum number of guests
 
         # Initialize data for guest suffixes and cached users
-        self.data.set_cache(data=[], key='guest_suffixes', do_not_overwrite=True)
+        self.data.set_cache_item(data=[], key='guest_suffixes', do_not_overwrite=True)
-        self.data.set_cache(data={}, key='cached_users', do_not_overwrite=True)
+        self.data.set_cache_item(data={}, key='cached_users', do_not_overwrite=True)
 
     def is_logged_in(self, user_name: str) -> bool:
         """
@@ -138,9 +138,9 @@ class UserAccountManagement(BaseUser):
                 # If the user is logged in, check if they are a guest.
                 if is_guest(user):
                     # Update the guest suffix cache if the user is a guest.
-                    guest_suffixes = self.data.get_cache('guest_suffixes')
+                    guest_suffixes = self.data.get_cache_item(key='guest_suffixes') or []
-                    guest_suffixes.append(user_name[1])
+                    guest_suffixes.append(user_name.split('_')[1])
-                    self.data.set_cache(data=guest_suffixes, key='guest_suffixes')
+                    self.data.set_cache_item(data=guest_suffixes, key='guest_suffixes')
                 return True
             else:
                 # If the user is not logged in, remove their data from the cache.
@@ -159,7 +159,7 @@ class UserAccountManagement(BaseUser):
         :return: True if the password is correct, False otherwise.
         """
         # Retrieve the hashed password using DataCache
-        user_data = self.data.fetch_cached_rows(table='users', filter_vals=('user_name', username))
+        user_data = self.data.get_or_fetch_rows(cache_name='users', filter_vals=('user_name', username))
 
         if user_data is None or user_data.empty:
             return False
@@ -215,23 +215,24 @@ class UserAccountManagement(BaseUser):
 
         :param enforcement: 'soft' or 'hard' - Determines how strictly users are logged out.
         """
-        if enforcement == 'soft':
+        # if enforcement == 'soft':
-            self._soft_log_out_all_users()
+        #     self._soft_log_out_all_users()
-        elif enforcement == 'hard':
+        # elif enforcement == 'hard':
-            # Clear all user-related entries from the cache
+        #     # Clear all user-related entries from the cache
-            for index, row in self.data.cache.iterrows():
+        #     for index, row in self.data.cache.iterrows():
-                if 'user_name' in row:
+        #         if 'user_name' in row:
-                    self._remove_user_from_memory(row['user_name'])
+        #             self._remove_user_from_memory(row['user_name'])
-
+        #
-            df = self.data.fetch_cached_rows(table='users', filter_vals=('status', 'logged_in'))
+        #     df = self.data.get_or_fetch_rows(cache_name='users', filter_vals=('status', 'logged_in'))
-            if df is not None:
+        #     if df is not None:
-                df = df[df.user_name != 'guest']
+        #         df = df[df.user_name != 'guest']
-
+        #
-                # Update the status of all logged-in users to 'logged_out'
+        #         # Update the status of all logged-in users to 'logged_out'
-                for user_name in df.user_name.values:
+        #         for user_name in df.user_name.values:
-                    self.modify_user_data(username=user_name, field_name='status', new_data='logged_out')
+        #             self.modify_user_data(username=user_name, field_name='status', new_data='logged_out')
-        else:
+        # else:
-            raise ValueError("Invalid enforcement type. Use 'soft' or 'hard'.")
+        #     raise ValueError("Invalid enforcement type. Use 'soft' or 'hard'.")
+        pass
 
     def _soft_log_out_all_users(self) -> None:
         """
@@ -250,7 +251,7 @@ class UserAccountManagement(BaseUser):
         :return: True if the attribute is already taken, False otherwise.
         """
         # Use DataCache to check if the attribute is taken
-        return self.data.is_attr_taken(table='users', attr=attr, val=val)
+        return self.data.is_attr_taken(cache_name='users', attr=attr, val=val)
 
     def create_unique_guest_name(self) -> str | None:
         """
@@ -258,12 +259,16 @@ class UserAccountManagement(BaseUser):
 
         :return: A unique guest username or None if the guest limit is reached.
         """
-        guest_suffixes = self.data.get_cache('guest_suffixes')
+        guest_suffixes = self.data.get_cache_item(key='guest_suffixes') or []
-        if len(guest_suffixes) > self.max_guests:
+        if len(guest_suffixes) >= self.max_guests:
             return None
-        suffix = random.choice(range(0, (self.max_guests * 9)))
+
+        suffix = random.choice(range(0, self.max_guests * 9))
         while suffix in guest_suffixes:
-            suffix = random.choice(range(0, (self.max_guests * 9)))
+            suffix = random.choice(range(0, self.max_guests * 9))
+
+        guest_suffixes.append(suffix)
+        self.data.set_cache_item(key='guest_suffixes', data=guest_suffixes)
         return f'guest_{suffix}'
 
     def create_guest(self) -> str | None:
@@ -292,7 +297,7 @@ class UserAccountManagement(BaseUser):
             raise ValueError("Attributes must be a tuple of single key-value pair dictionaries.")
 
         # Retrieve the default user template from the database using DataCache
-        default_user = self.data.fetch_cached_rows(table='users', filter_vals=('user_name', 'guest'))
+        default_user = self.data.get_or_fetch_rows(cache_name='users', filter_vals=('user_name', 'guest'))
 
         if default_user is None or default_user.empty:
             raise ValueError("Default user template not found in the database.")
@@ -306,7 +311,7 @@ class UserAccountManagement(BaseUser):
         default_user = default_user.drop(columns='id')
 
         # Insert the modified user data into the database, skipping cache insertion
-        self.data.insert_data(df=default_user, table="users", skip_cache=True)
+        self.data.insert_df(df=default_user, cache_name="users", skip_cache=True)
 
     def create_new_user(self, username: str, email: str, password: str) -> bool:
         """
@@ -456,7 +461,7 @@ class UserIndicatorManagement(UserExchangeManagement):
         user_id = self.get_id(user_name)
 
         # Fetch the indicators from the database using DataCache
-        df = self.data.fetch_cached_rows(table='indicators', filter_vals=('creator', user_id))
+        df = self.data.get_or_fetch_rows(cache_name='indicators', filter_vals=('creator', user_id))
 
         # If indicators are found, process the JSON fields
         if df is not None and not df.empty:
@@ -481,8 +486,8 @@ class UserIndicatorManagement(UserExchangeManagement):
                       indicator['kind'], src_string, prop_string)
             columns = ('creator', 'name', 'visible', 'kind', 'source', 'properties')
 
-            # Insert the row into the database using DataCache
+            # Insert the row into the database and cache using DataCache
-            self.data.insert_row(table='indicators', columns=columns, values=values)
+            self.data.insert_row(cache_name='indicators', columns=columns, values=values)
 
     def remove_indicator(self, indicator_name: str, user_name: str) -> None:
         """
@@ -495,7 +500,7 @@ class UserIndicatorManagement(UserExchangeManagement):
         self.data.remove_row(
             filter_vals=('name', indicator_name),
             additional_filter=('creator', user_id),
-            table='indicators'
+            cache_name='indicators'
         )
 
     def get_chart_view(self, user_name: str, prop: str | None = None):

src/trade.py

@@ -1,6 +1,6 @@
 import json
 import uuid
-
+from Users import Users
 import requests
 from datetime import datetime
 
@@ -267,7 +267,7 @@ class Trade:
 
 
 class Trades:
-    def __init__(self, users):
+    def __init__(self, users: Users):
         """
         This class receives, executes, tracks and stores all active_trades.
         :param users: <Users> A class that maintains users each user may have trades.
@@ -291,10 +291,10 @@ class Trades:
         self.stats = {'num_trades': 0, 'total_position': 0, 'total_position_value': 0}
 
         # Load all trades.
-        loaded_trades = users.get_all_active_user_trades()
+        # loaded_trades = users.get_all_active_user_trades()
-        if loaded_trades is not None:
+        # if loaded_trades is not None:
-            # Create the active_trades loaded from file.
+        #     # Create the active_trades loaded from file.
-            self.load_trades(loaded_trades)
+        #     self.load_trades(loaded_trades)
 
     def connect_exchanges(self, exchanges):
         """

tests/test_DataCache.py

@@ -1,5 +1,7 @@
+import time
 import pytz
-from DataCache_v2 import DataCache, timeframe_to_timedelta, estimate_record_count
+from DataCache_v3 import DataCache, timeframe_to_timedelta, estimate_record_count, InMemoryCache, DataCacheBase, \
+    SnapshotDataCache
 from ExchangeInterface import ExchangeInterface
 import unittest
 import pandas as pd
@@ -191,7 +193,7 @@ class DataGenerator:
         return dt_obj
 
 
-class TestDataCacheV2(unittest.TestCase):
+class TestDataCache(unittest.TestCase):
     def setUp(self):
         # Set up database and exchanges
         self.exchanges = ExchangeInterface()
@@ -229,11 +231,16 @@ class TestDataCacheV2(unittest.TestCase):
             key TEXT PRIMARY KEY,
             data TEXT NOT NULL
             )"""
-        sql_create_table_5 = f"""
+        sql_create_table_5 = """
-            CREATE TABLE IF NOT EXISTS users (
+        CREATE TABLE IF NOT EXISTS users (
-            users_data TEXT PRIMARY KEY,
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            data TEXT NOT NULL
+            user_name TEXT,
-            )"""
+            age INTEGER,
+            users_data TEXT,
+            data TEXT,
+            password TEXT  -- Moved to a new line and added a comma after 'data'
+        )
+        """
 
         with SQLite(db_file=self.db_file) as con:
             con.execute(sql_create_table_1)
@@ -251,44 +258,188 @@ class TestDataCacheV2(unittest.TestCase):
         if os.path.exists(self.db_file):
             os.remove(self.db_file)
 
-    def test_set_cache(self):
+    def test_InMemoryCache(self):
-        print('\nTesting set_cache() method without no-overwrite flag:')
+        # Step 1: Create a cache with a limit of 2 items and 'evict' policy
-        self.data.set_cache(data='data', key=self.key)
+        print("Creating a cache with a limit of 2 items and 'evict' policy.")
+        cached_users = InMemoryCache(limit=2, eviction_policy='evict')
 
-        # Access the cache data using the DataFrame structure
+        # Step 2: Set some items in the cache.
-        cached_value = self.data.get_cache(key=self.key)
+        print("Setting 'user_bob' in the cache with an expiration of 10 seconds.")
-        self.assertEqual(cached_value, 'data')
+        cached_users.set_item("user_bob", "{password:'BobPass'}", expire_delta=dt.timedelta(seconds=10))
-        print(' - Set data without no-overwrite flag passed.')
 
-        print('Testing set_cache() once again with new data without no-overwrite flag:')
+        print("Setting 'user_alice' in the cache with an expiration of 20 seconds.")
-        self.data.set_cache(data='more_data', key=self.key)
+        cached_users.set_item("user_alice", "{password:'AlicePass'}", expire_delta=dt.timedelta(seconds=20))
 
-        # Access the updated cache data
+        # Step 3: Retrieve 'user_bob' from the cache
-        cached_value = self.data.get_cache(key=self.key)
+        print("Retrieving 'user_bob' from the cache.")
-        self.assertEqual(cached_value, 'more_data')
+        retrieved_item = cached_users.get_item('user_bob')
-        print(' - Set data with new data without no-overwrite flag passed.')
+        print(f"Retrieved: {retrieved_item}")
+        assert retrieved_item == "{password:'BobPass'}", "user_bob should have been retrieved successfully."
 
-        print('Testing set_cache() method once again with more data with no-overwrite flag set:')
+        # Step 4: Add another item, causing the oldest item to be evicted
-        self.data.set_cache(data='even_more_data', key=self.key, do_not_overwrite=True)
+        print("Adding 'user_billy' to the cache, which should evict 'user_bob' due to the limit.")
+        cached_users.set_item("user_billy", "{password:'BillyPass'}")
 
-        # Since do_not_overwrite is True, the cached data should not change
+        # Step 5: Attempt to retrieve the evicted item 'user_bob'
-        cached_value = self.data.get_cache(key=self.key)
+        print("Attempting to retrieve the evicted item 'user_bob'.")
-        self.assertEqual(cached_value, 'more_data')
+        evicted_item = cached_users.get_item('user_bob')
-        print(' - Set data with no-overwrite flag passed.')
+        print(f"Evicted Item: {evicted_item}")
+        assert evicted_item is None, "user_bob should have been evicted from the cache."
 
-    def test_cache_exists(self):
+        # Step 6: Retrieve the current items in the cache
-        print('Testing cache_exists() method:')
+        print("Retrieving all current items in the cache after eviction.")
+        all_items = cached_users.get_all_items()
+        print("Current items in cache:\n", all_items)
+        assert "user_alice" in all_items['key'].values, "user_alice should still be in the cache."
+        assert "user_billy" in all_items['key'].values, "user_billy should still be in the cache."
 
-        # Check that the cache does not contain the key before setting it
+        # Step 7: Simulate waiting for 'user_alice' to expire (assuming 20 seconds pass)
-        self.assertFalse(self.data.cache_exists(key=self.key))
+        print("Simulating time passing to expire 'user_alice' (20 seconds).")
-        print(' - Check for non-existent data passed.')
+        time.sleep(20)  # This is to simulate the passage of time; in real tests, you may mock datetime.
 
-        # Set the cache with a DataFrame containing the key-value pair
+        # Step 8: Clean expired items from the cache
-        self.data.set_cache(data='data', key=self.key)
+        print("Cleaning expired items from the cache.")
+        cached_users.clean_expired_items()
 
-        # Check that the cache now contains the key
+        # Step 9: Retrieve the current items in the cache after cleaning expired items
-        self.assertTrue(self.data.cache_exists(key=self.key))
+        print("Retrieving all current items in the cache after cleaning expired items.")
-        print(' - Check for existent data passed.')
+        all_items_after_cleaning = cached_users.get_all_items()
+        print("Current items in cache after cleaning:\n", all_items_after_cleaning)
+        assert "user_alice" not in all_items_after_cleaning[
+            'key'].values, "user_alice should have been expired and removed from the cache."
+        assert "user_billy" in all_items_after_cleaning['key'].values, "user_billy should still be in the cache."
+
+        # Step 10: Check if 'user_billy' still exists as it should not expire
+        print("Checking if 'user_billy' still exists in the cache (it should not have expired).")
+        user_billy_item = cached_users.get_item('user_billy')
+        print(f"'user_billy' still exists: {user_billy_item}")
+        assert user_billy_item == "{password:'BillyPass'}", "user_billy should still exist in the cache."
+
+    def test_DataCacheBase(self):
+        # Step 1: Create a DataCacheBase instance
+        print("Creating a DataCacheBase instance.")
+        cache_manager = DataCacheBase()
+
+        # Step 2: Set some items in 'my_cache'. The cache is created automatically with limit 2 and 'evict' policy.
+        print("Setting 'key1' in 'my_cache' with an expiration of 10 seconds.")
+        cache_manager.set_cache_item('key1', 'data1', expire_delta=dt.timedelta(seconds=10), cache_name='my_cache',
+                                     limit=2, eviction_policy='evict')
+
+        print("Setting 'key2' in 'my_cache' with an expiration of 20 seconds.")
+        cache_manager.set_cache_item('key2', 'data2', expire_delta=dt.timedelta(seconds=20), cache_name='my_cache')
+
+        # Step 3: Set some items in 'second_cache'. The cache is created automatically with limit 3 and 'deny' policy.
+        print("Setting 'keyA' in 'second_cache' with an expiration of 15 seconds.")
+        cache_manager.set_cache_item('keyA', 'dataA', expire_delta=dt.timedelta(seconds=15), cache_name='second_cache',
+                                     limit=3, eviction_policy='deny')
+
+        print("Setting 'keyB' in 'second_cache' with an expiration of 30 seconds.")
+        cache_manager.set_cache_item('keyB', 'dataB', expire_delta=dt.timedelta(seconds=30), cache_name='second_cache')
+
+        print("Setting 'keyC' in 'second_cache' with no expiration.")
+        cache_manager.set_cache_item('keyC', 'dataC', cache_name='second_cache')
+
+        # Step 4: Add another item to 'my_cache', causing the oldest item to be evicted.
+        print("Adding 'key3' to 'my_cache', which should evict 'key1' due to the limit.")
+        cache_manager.set_cache_item('key3', 'data3', cache_name='my_cache')
+
+        # Step 5: Attempt to retrieve the evicted item 'key1' from 'my_cache'.
+        print("Attempting to retrieve the evicted item 'key1' from 'my_cache'.")
+        evicted_item = cache_manager.get_cache_item('key1', cache_name='my_cache')
+        print(f"Evicted Item from 'my_cache': {evicted_item}")
+        assert evicted_item is None, "'key1' should have been evicted from 'my_cache'."
+
+        # Step 6: Retrieve all current items in both caches before cleaning.
+        print("Retrieving all current items in 'my_cache' before cleaning.")
+        all_items_my_cache = cache_manager.get_all_cache_items('my_cache')
+        print("Current items in 'my_cache':\n", all_items_my_cache)
+
+        print("Retrieving all current items in 'second_cache' before cleaning.")
+        all_items_second_cache = cache_manager.get_all_cache_items('second_cache')
+        print("Current items in 'second_cache':\n", all_items_second_cache)
+
+        # Step 7: Simulate time passing to expire 'key2' in 'my_cache' and 'keyA' in 'second_cache'.
+        print("Simulating time passing to expire 'key2' in 'my_cache' (20 seconds)"
+              " and 'keyA' in 'second_cache' (15 seconds).")
+        time.sleep(20)  # Simulate the passage of time; in real tests, you may mock datetime.
+
+        # Step 8: Clean expired items in all caches
+        print("Cleaning expired items in all caches.")
+        cache_manager.clean_expired_items()
+
+        # Step 9: Verify the cleaning of expired items in 'my_cache'.
+        print("Retrieving all current items in 'my_cache' after cleaning expired items.")
+        all_items_after_cleaning_my_cache = cache_manager.get_all_cache_items('my_cache')
+        print("Items in 'my_cache' after cleaning:\n", all_items_after_cleaning_my_cache)
+        assert 'key2' not in all_items_after_cleaning_my_cache[
+            'key'].values, "'key2' should have been expired and removed from 'my_cache'."
+        assert 'key3' in all_items_after_cleaning_my_cache['key'].values, "'key3' should still be in 'my_cache'."
+
+        # Step 10: Verify the cleaning of expired items in 'second_cache'.
+        print("Retrieving all current items in 'second_cache' after cleaning expired items.")
+        all_items_after_cleaning_second_cache = cache_manager.get_all_cache_items('second_cache')
+        print("Items in 'second_cache' after cleaning:\n", all_items_after_cleaning_second_cache)
+        assert 'keyA' not in all_items_after_cleaning_second_cache[
+            'key'].values, "'keyA' should have been expired and removed from 'second_cache'."
+        assert 'keyB' in all_items_after_cleaning_second_cache[
+            'key'].values, "'keyB' should still be in 'second_cache'."
+        assert 'keyC' in all_items_after_cleaning_second_cache[
+            'key'].values, "'keyC' should still be in 'second_cache' since it has no expiration."
+
+    def test_SnapshotDataCache(self):
+        # Step 1: Create a SnapshotDataCache instance
+        print("Creating a SnapshotDataCache instance.")
+        snapshot_cache_manager = SnapshotDataCache()
+
+        # Step 2: Create an in-memory cache with a limit of 2 items and 'evict' policy
+        print("Creating an in-memory cache named 'my_cache' with a limit of 2 items and 'evict' policy.")
+        snapshot_cache_manager.create_cache('my_cache', cache_type=InMemoryCache, limit=2, eviction_policy='evict')
+
+        # Step 3: Set some items in the cache
+        print("Setting 'key1' in 'my_cache' with an expiration of 10 seconds.")
+        snapshot_cache_manager.set_cache_item(key='key1', data='data1', expire_delta=dt.timedelta(seconds=10),
+                                              cache_name='my_cache')
+
+        print("Setting 'key2' in 'my_cache' with an expiration of 20 seconds.")
+        snapshot_cache_manager.set_cache_item(key='key2', data='data2', expire_delta=dt.timedelta(seconds=20),
+                                              cache_name='my_cache')
+
+        # Step 4: Take a snapshot of the current state of 'my_cache'
+        print("Taking a snapshot of the current state of 'my_cache'.")
+        snapshot_cache_manager.snapshot_cache('my_cache')
+
+        # Step 5: Add another item, causing the oldest item to be evicted
+        print("Adding 'key3' to 'my_cache', which should evict 'key1' due to the limit.")
+        snapshot_cache_manager.set_cache_item(key='key3', data='data3', cache_name='my_cache')
+
+        # Step 6: Retrieve the most recent snapshot of 'my_cache'
+        print("Retrieving the most recent snapshot of 'my_cache'.")
+        snapshot = snapshot_cache_manager.get_snapshot('my_cache')
+        print(f"Snapshot Data:\n{snapshot}")
+
+        # Assert that the snapshot contains 'key1' and 'key2', but not 'key3'
+        assert 'key1' in snapshot['key'].values, "'key1' should be in the snapshot."
+        assert 'key2' in snapshot['key'].values, "'key2' should be in the snapshot."
+        assert 'key3' not in snapshot[
+            'key'].values, "'key3' should not be in the snapshot as it was added after the snapshot."
+
+        # Step 7: List all available snapshots with their timestamps
+        print("Listing all available snapshots with their timestamps.")
+        snapshots_list = snapshot_cache_manager.list_snapshots()
+        print(f"Snapshots List: {snapshots_list}")
+
+        # Assert that the snapshot list contains 'my_cache'
+        assert 'my_cache' in snapshots_list, "'my_cache' should be in the snapshots list."
+        assert isinstance(snapshots_list['my_cache'], str), "The snapshot for 'my_cache' should have a timestamp."
+
+        # Additional validation: Ensure 'key3' is present in the live cache but not in the snapshot
+        print("Ensuring 'key3' is present in the live 'my_cache'.")
+        live_cache_items = snapshot_cache_manager.get_all_cache_items('my_cache')
+        print(f"Live 'my_cache' items after adding 'key3':\n{live_cache_items}")
+        assert 'key3' in live_cache_items['key'].values, "'key3' should be in the live cache."
+
+        # Ensure the live cache does not contain 'key1'
+        assert 'key1' not in live_cache_items['key'].values, "'key1' should have been evicted from the live cache."
 
     def test_update_candle_cache(self):
         print('Testing update_candle_cache() method:')
@@ -296,18 +447,18 @@ class TestDataCacheV2(unittest.TestCase):
         # Initialize the DataGenerator with the 5-minute timeframe
         data_gen = DataGenerator('5m')
 
-        # Create initial DataFrame and insert into cache
+        # Create initial DataFrame and insert it into the cache
         df_initial = data_gen.create_table(num_rec=3, start=dt.datetime(2024, 8, 9, 0, 0, 0, tzinfo=dt.timezone.utc))
         print(f'Inserting this table into cache:\n{df_initial}\n')
-        self.data.set_cache(data=df_initial, key=self.key)
+        self.data.set_cache_item(key=self.key, data=df_initial, cache_name='candles')
 
-        # Create new DataFrame to be added to cache
+        # Create new DataFrame to be added to the cache
         df_new = data_gen.create_table(num_rec=3, start=dt.datetime(2024, 8, 9, 0, 15, 0, tzinfo=dt.timezone.utc))
         print(f'Updating cache with this table:\n{df_new}\n')
         self.data._update_candle_cache(more_records=df_new, key=self.key)
 
-        # Retrieve the resulting DataFrame from cache
+        # Retrieve the resulting DataFrame from the cache
-        result = self.data.get_cache(key=self.key)
+        result = self.data.get_cache_item(key=self.key, cache_name='candles')
         print(f'The resulting table in cache is:\n{result}\n')
 
         # Create the expected DataFrame
@@ -316,8 +467,7 @@ class TestDataCacheV2(unittest.TestCase):
 
         # Assert that the open_time values in the result match those in the expected DataFrame, in order
         assert result['open_time'].tolist() == expected['open_time'].tolist(), \
-            f"open_time values in result are {result['open_time'].tolist()}" \
+            f"open_time values in result are {result['open_time'].tolist()} expected {expected['open_time'].tolist()}"
-            f" but expected {expected['open_time'].tolist()}"
 
         print(f'The result open_time values match:\n{result["open_time"].tolist()}\n')
         print(' - Update cache with new records passed.')
@@ -325,32 +475,25 @@ class TestDataCacheV2(unittest.TestCase):
     def test_update_cached_dict(self):
         print('Testing update_cached_dict() method:')
 
-        # Set an empty dictionary in the cache for the specified key
+        # Step 1: Set an empty dictionary in the cache for the specified key
-        self.data.set_cache(data={}, key=self.key)
+        print(f'Setting an empty dictionary in the cache with key: {self.key}')
+        self.data.set_cache_item(data={}, key=self.key)
 
-        # Update the cached dictionary with a new key-value pair
+        # Step 2: Update the cached dictionary with a new key-value pair
-        self.data.update_cached_dict(cache_key=self.key, dict_key='sub_key', data='value')
+        print(f'Updating the cached dictionary with key: {self.key}, adding sub_key="sub_key" with value="value".')
+        self.data.update_cached_dict(cache_name='default_cache', cache_key=self.key, dict_key='sub_key', data='value')
 
-        # Retrieve the updated cache
+        # Step 3: Retrieve the updated cache
-        cache = self.data.get_cache(key=self.key)
+        print(f'Retrieving the updated dictionary from the cache with key: {self.key}')
+        cache = self.data.get_cache_item(key=self.key)
 
-        # Verify that the 'sub_key' in the cached dictionary has the correct value
+        # Step 4: Verify that the 'sub_key' in the cached dictionary has the correct value
+        print(f'Verifying that "sub_key" in the cached dictionary has the value "value".')
+        self.assertIsInstance(cache, dict, "The cache should be a dictionary.")
+        self.assertIn('sub_key', cache, "The 'sub_key' should be present in the cached dictionary.")
         self.assertEqual(cache['sub_key'], 'value')
         print(' - Update dictionary in cache passed.')
 
-    def test_get_cache(self):
-        print('Testing get_cache() method:')
-
-        # Set some data into the cache
-        self.data.set_cache(data='data', key=self.key)
-
-        # Retrieve the cached data using the get_cache method
-        result = self.data.get_cache(key=self.key)
-
-        # Verify that the result matches the data we set
-        self.assertEqual(result, 'data')
-        print(' - Retrieve data passed.')
-
     def _test_get_records_since(self, set_cache=True, set_db=True, query_offset=None, num_rec=None, ex_details=None,
                                 simulate_scenarios=None):
         """
@@ -371,94 +514,70 @@ class TestDataCacheV2(unittest.TestCase):
 
         print('Testing get_records_since() method:')
 
-        # Use provided ex_details or fallback to the class attribute.
         ex_details = ex_details or self.ex_details
-        # Generate a data/database key using exchange details.
         key = f'{ex_details[0]}_{ex_details[1]}_{ex_details[2]}'
-
-        # Set default number of records if not provided.
         num_rec = num_rec or 12
-        table_timeframe = ex_details[1]  # Extract timeframe from exchange details.
+        table_timeframe = ex_details[1]
 
-        # Initialize DataGenerator with the given timeframe.
         data_gen = DataGenerator(table_timeframe)
 
         if simulate_scenarios == 'not_enough_data':
-            # Set query_offset to a time earlier than the start of the table data.
             query_offset = (num_rec + 5) * data_gen.timeframe_amount
         else:
-            # Default to querying for 1 record length less than the table duration.
             query_offset = query_offset or (num_rec - 1) * data_gen.timeframe_amount
 
         if simulate_scenarios == 'incomplete_data':
-            # Set start time to generate fewer records than required.
             start_time_for_data = data_gen.x_time_ago(num_rec * data_gen.timeframe_amount)
-            num_rec = 5  # Set a smaller number of records to simulate incomplete data.
+            num_rec = 5
         else:
-            # No specific start time for data generation.
             start_time_for_data = None
 
-        # Create the initial data table.
         df_initial = data_gen.create_table(num_rec, start=start_time_for_data)
 
         if simulate_scenarios == 'missing_section':
-            # Simulate missing section in the data by dropping records.
             df_initial = data_gen.generate_missing_section(df_initial, drop_start=2, drop_end=5)
 
-        # Convert 'open_time' to datetime for better readability.
         temp_df = df_initial.copy()
         temp_df['open_time'] = pd.to_datetime(temp_df['open_time'], unit='ms')
         print(f'Table Created:\n{temp_df}')
 
         if set_cache:
-            # Insert the generated table into the cache.
+            print('Ensuring the cache exists and then inserting table into the cache.')
-            print('Inserting table into the cache.')
+            self.data.set_cache_item(data=df_initial, key=key, cache_name='candles')
-            self.data.set_cache(data=df_initial, key=key)
 
         if set_db:
-            # Insert the generated table into the database.
             print('Inserting table into the database.')
             with SQLite(self.db_file) as con:
                 df_initial.to_sql(key, con, if_exists='replace', index=False)
 
-        # Calculate the start time for querying the records.
         start_datetime = data_gen.x_time_ago(query_offset)
-
-        # Ensure start_datetime is timezone-aware (UTC).
         if start_datetime.tzinfo is None:
             start_datetime = start_datetime.replace(tzinfo=dt.timezone.utc)
 
-        # Defaults to current time if not provided to get_records_since()
         query_end_time = dt.datetime.utcnow().replace(tzinfo=dt.timezone.utc)
         print(f'Requesting records from {start_datetime} to {query_end_time}')
 
-        # Query the records since the calculated start time.
         result = self.data.get_records_since(start_datetime=start_datetime, ex_details=ex_details)
 
-        # Filter the initial data table to match the query time.
         expected = df_initial[df_initial['open_time'] >= data_gen.unix_time_millis(start_datetime)].reset_index(
             drop=True)
         temp_df = expected.copy()
         temp_df['open_time'] = pd.to_datetime(temp_df['open_time'], unit='ms')
         print(f'Expected table:\n{temp_df}')
 
-        # Print the result from the query for comparison.
         temp_df = result.copy()
         temp_df['open_time'] = pd.to_datetime(temp_df['open_time'], unit='ms')
         print(f'Resulting table:\n{temp_df}')
 
         if simulate_scenarios in ['not_enough_data', 'incomplete_data', 'missing_section']:
-            # Check that the result has more rows than the expected incomplete data.
             assert result.shape[0] > expected.shape[
                 0], "Result has fewer or equal rows compared to the incomplete data."
             print("\nThe returned DataFrame has filled in the missing data!")
         else:
-            # Ensure the result and expected dataframes match in shape and content.
             assert result.shape == expected.shape, f"Shape mismatch: {result.shape} vs {expected.shape}"
             pd.testing.assert_series_equal(result['open_time'], expected['open_time'], check_dtype=False)
             print("\nThe DataFrames have the same shape and the 'open_time' columns match.")
 
-        # Verify that the oldest timestamp in the result is within the allowed time difference.
         oldest_timestamp = pd.to_datetime(result['open_time'].min(), unit='ms').tz_localize('UTC')
         time_diff = oldest_timestamp - start_datetime
         max_allowed_time_diff = dt.timedelta(**{data_gen.timeframe_unit: data_gen.timeframe_amount})
@@ -469,7 +588,6 @@ class TestDataCacheV2(unittest.TestCase):
 
         print(f'The first timestamp is {time_diff} from {start_datetime}')
 
-        # Verify that the newest timestamp in the result is within the allowed time difference.
         newest_timestamp = pd.to_datetime(result['open_time'].max(), unit='ms').tz_localize('UTC')
         time_diff_end = abs(query_end_time - newest_timestamp)
 
@@ -505,6 +623,9 @@ class TestDataCacheV2(unittest.TestCase):
 
     def test_other_timeframes(self):
         print('\nTest get_records_since with a different timeframe')
+        if 'candles' not in self.data.caches:
+            self.data.create_cache(cache_name='candles')
+
         ex_details = ['BTC/USD', '15m', 'binance', 'test_guy']
         start_datetime = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=2)
         # Query the records since the calculated start time.
@@ -573,37 +694,53 @@ class TestDataCacheV2(unittest.TestCase):
     def test_remove_row(self):
         print('Testing remove_row() method:')
 
-        # Insert data into the cache with the expected columns
+        # Create a DataFrame to insert as the data
-        df = pd.DataFrame({
+        user_data = pd.DataFrame({
-            'key': [self.key],
+            'user_name': ['test_user'],
-            'data': ['test_data']
+            'password': ['test_password']
         })
-        self.data.set_cache(data='test_data', key=self.key)
+
+        # Insert data into the cache
+        self.data.set_cache_item(
+            cache_name='users',
+            key='user1',
+            data=user_data
+        )
 
         # Ensure the data is in the cache
-        self.assertTrue(self.data.cache_exists(self.key), "Data was not correctly inserted into the cache.")
+        cache_item = self.data.get_cache_item('user1', 'users')
+        self.assertIsNotNone(cache_item, "Data was not correctly inserted into the cache.")
+
+        # The cache_item is a DataFrame, so we access the 'user_name' column directly
+        self.assertEqual(cache_item['user_name'].iloc[0], 'test_user', "Inserted data is incorrect.")
 
         # Remove the row from the cache only (soft delete)
-        self.data.remove_row(table='test_table_2', filter_vals=('key', self.key), remove_from_db=False)
+        self.data.remove_row(cache_name='users', filter_vals=('user_name', 'test_user'), remove_from_db=False)
 
         # Verify the row has been removed from the cache
-        self.assertFalse(self.data.cache_exists(self.key), "Row was not correctly removed from the cache.")
+        cache_item = self.data.get_cache_item('user1', 'users')
+        self.assertIsNone(cache_item, "Row was not correctly removed from the cache.")
 
         # Reinsert the data for hard delete test
-        self.data.set_cache(data='test_data', key=self.key)
+        self.data.set_cache_item(
+            cache_name='users',
+            key='user1',
+            data=user_data
+        )
 
         # Mock database delete by adding the row to the database
-        self.data.db.insert_row(table='test_table_2', columns=('key', 'data'), values=(self.key, 'test_data'))
+        self.data.db.insert_row(table='users', columns=('user_name', 'password'), values=('test_user', 'test_password'))
 
         # Remove the row from both cache and database (hard delete)
-        self.data.remove_row(table='test_table_2', filter_vals=('key', self.key), remove_from_db=True)
+        self.data.remove_row(cache_name='users', filter_vals=('user_name', 'test_user'), remove_from_db=True)
 
         # Verify the row has been removed from the cache
-        self.assertFalse(self.data.cache_exists(self.key), "Row was not correctly removed from the cache.")
+        cache_item = self.data.get_cache_item('user1', 'users')
+        self.assertIsNone(cache_item, "Row was not correctly removed from the cache.")
 
         # Verify the row has been removed from the database
         with SQLite(self.db_file) as con:
-            result = pd.read_sql(f'SELECT * FROM test_table_2 WHERE key="{self.key}"', con)
+            result = pd.read_sql(f'SELECT * FROM users WHERE user_name="test_user"', con)
             self.assertTrue(result.empty, "Row was not correctly removed from the database.")
 
         print(' - Remove row from cache and database passed.')
@@ -662,80 +799,164 @@ class TestDataCacheV2(unittest.TestCase):
 
         print(' - All estimate_record_count() tests passed.')
 
-    def test_fetch_cached_rows(self):
+    def test_get_or_fetch_rows(self):
-        print('Testing fetch_cached_rows() method:')
 
-        # Set up mock data in the cache
+        # Create a mock table in the cache with multiple entries
-        df = pd.DataFrame({
+        df1 = pd.DataFrame({
-            'table': ['test_table_2'],
+            'user_name': ['billy'],
-            'key': ['test_key'],
+            'password': ['1234'],
-            'data': ['test_data']
+            'exchanges': [['ex1', 'ex2', 'ex3']]
         })
-        self.data.cache = pd.concat([self.data.cache, df])
 
-        # Test fetching from cache
+        df2 = pd.DataFrame({
-        result = self.data.fetch_cached_rows('test_table_2', ('key', 'test_key'))
+            'user_name': ['john'],
+            'password': ['5678'],
+            'exchanges': [['ex4', 'ex5', 'ex6']]
+        })
+
+        df3 = pd.DataFrame({
+            'user_name': ['alice'],
+            'password': ['91011'],
+            'exchanges': [['ex7', 'ex8', 'ex9']]
+        })
+
+        # Insert these DataFrames into the 'users' cache
+        self.data.create_cache('users', cache_type=InMemoryCache)
+        self.data.set_cache_item(key='user_billy', data=df1, cache_name='users')
+        self.data.set_cache_item(key='user_john', data=df2, cache_name='users')
+        self.data.set_cache_item(key='user_alice', data=df3, cache_name='users')
+
+        print('Testing get_or_fetch_rows() method:')
+
+        # Test fetching an existing user from the cache
+        result = self.data.get_or_fetch_rows('users', ('user_name', 'billy'))
         self.assertIsInstance(result, pd.DataFrame, "Failed to fetch DataFrame from cache")
         self.assertFalse(result.empty, "The fetched DataFrame is empty")
-        self.assertEqual(result.iloc[0]['data'], 'test_data', "Incorrect data fetched from cache")
+        self.assertEqual(result.iloc[0]['password'], '1234', "Incorrect data fetched from cache")
 
-        # Test fetching from database (assuming the method calls it)
+        # Test fetching another user from the cache
-        # Here we would typically mock the database call
+        result = self.data.get_or_fetch_rows('users', ('user_name', 'john'))
-        # But since we're not doing I/O, we will skip that part
+        self.assertIsInstance(result, pd.DataFrame, "Failed to fetch DataFrame from cache")
-        print(' - Fetch from cache and database simulated.')
+        self.assertFalse(result.empty, "The fetched DataFrame is empty")
+        self.assertEqual(result.iloc[0]['password'], '5678', "Incorrect data fetched from cache")
+
+        # Test fetching a user that does not exist in the cache
+        result = self.data.get_or_fetch_rows('users', ('user_name', 'non_existent_user'))
+
+        # Check if result is None (indicating that no data was found)
+        self.assertIsNone(result, "Expected result to be None for a non-existent user")
+
+        print(' - Fetching rows from cache passed.')
 
     def test_is_attr_taken(self):
-        print('Testing is_attr_taken() method:')
+        # Create a cache named 'users'
+        self.data.create_cache('users', cache_type=InMemoryCache)
 
-        # Set up mock data in the cache
+        # Create mock data for three users
-        df = pd.DataFrame({
+        user_data_1 = pd.DataFrame({
-            'table': ['users'],
+            'user_name': ['billy'],
-            'user_name': ['test_user'],
+            'password': ['1234'],
-            'data': ['test_data']
+            'exchanges': [['ex1', 'ex2', 'ex3']]
+        })
+        user_data_2 = pd.DataFrame({
+            'user_name': ['john'],
+            'password': ['5678'],
+            'exchanges': [['ex1', 'ex2', 'ex4']]
+        })
+        user_data_3 = pd.DataFrame({
+            'user_name': ['alice'],
+            'password': ['abcd'],
+            'exchanges': [['ex5', 'ex6', 'ex7']]
         })
-        self.data.cache = pd.concat([self.data.cache, df])
 
-        # Test for existing attribute
+        # Insert mock data into the cache
-        result = self.data.is_attr_taken('users', 'user_name', 'test_user')
+        self.data.set_cache_item('user1', user_data_1, cache_name='users')
-        self.assertTrue(result, "Failed to detect existing attribute")
+        self.data.set_cache_item('user2', user_data_2, cache_name='users')
+        self.data.set_cache_item('user3', user_data_3, cache_name='users')
 
-        # Test for non-existing attribute
+        # Test when attribute value is taken
-        result = self.data.is_attr_taken('users', 'user_name', 'non_existing_user')
+        result_taken = self.data.is_attr_taken(cache_name='users', attr='user_name', val='billy')
-        self.assertFalse(result, "Incorrectly detected non-existing attribute")
+        self.assertTrue(result_taken, "Expected 'billy' to be taken, but it was not.")
 
-        print(' - All is_attr_taken() tests passed.')
+        # Test when attribute value is not taken
+        result_not_taken = self.data.is_attr_taken(cache_name='users', attr='user_name', val='charlie')
+        self.assertFalse(result_not_taken, "Expected 'charlie' not to be taken, but it was.")
 
-    def test_insert_data(self):
+    def test_insert_df(self):
-        print('Testing insert_data() method:')
+        print('Testing insert_df() method:')
 
         # Create a DataFrame to insert
         df = pd.DataFrame({
-            'key': ['new_key'],
+            'user_name': ['Alice'],
-            'data': ['new_data']
+            'age': [30],
+            'users_data': ['user_data_1'],
+            'data': ['additional_data'],
+            'password': ['1234']
         })
 
         # Insert data into the database and cache
-        self.data.insert_data(df=df, table='test_table_2')
+        self.data.insert_df(df=df, cache_name='users')
 
-        # Verify that the data was added to the cache
+        # Assume the database will return an auto-incremented ID starting at 1
-        cached_value = self.data.get_cache('new_key')
+        auto_incremented_id = 1
-        self.assertEqual(cached_value, 'new_data', "Failed to insert data into cache")
 
-        # Normally, we would also verify that the data was inserted into the database
+        # Verify that the data was added to the cache using the auto-incremented ID as the key
-        # This would typically be done with a mock database or by checking the database state directly
+        cached_df = self.data.get_cache_item(key=str(auto_incremented_id), cache_name='users')
-        print(' - Data insertion into cache and database simulated.')
+
+        # Check that the DataFrame in the cache matches the original DataFrame
+        pd.testing.assert_frame_equal(cached_df, df, check_dtype=False)
+
+        # Now, let's verify the data was inserted into the database
+        with SQLite(self.data.db.db_file) as conn:
+            # Query the users table for the inserted data
+            query_result = pd.read_sql_query(f"SELECT * FROM users WHERE id = {auto_incremented_id}", conn)
+
+        # Verify the database content matches the inserted DataFrame
+        expected_db_df = df.copy()
+        expected_db_df['id'] = auto_incremented_id  # Add the auto-incremented ID to the expected DataFrame
+        # Align column order
+        expected_db_df = expected_db_df[['id', 'user_name', 'age', 'users_data', 'data', 'password']]
+
+        # Check that the database DataFrame matches the expected DataFrame
+        pd.testing.assert_frame_equal(query_result, expected_db_df, check_dtype=False)
+
+        print(' - Data insertion into cache and database verified successfully.')
 
     def test_insert_row(self):
-        print('Testing insert_row() method:')
+        print("Testing insert_row() method:")
 
-        self.data.insert_row(table='test_table_2', columns=('key', 'data'), values=('test_key', 'test_data'))
+        # Define the cache name, columns, and values to insert
+        cache_name = 'users'
+        columns = ('user_name', 'age')
+        values = ('Alice', 30)
 
-        # Verify the row was inserted
+        # Create the cache first
-        with SQLite(self.db_file) as con:
+        self.data.create_cache(cache_name, cache_type=InMemoryCache)
-            result = pd.read_sql('SELECT * FROM test_table_2 WHERE key="test_key"', con)
-            self.assertFalse(result.empty, "Row was not inserted into the database.")
 
-        print(' - Insert row passed.')
+        # Insert a row into the cache and database without skipping the cache
+        self.data.insert_row(cache_name=cache_name, columns=columns, values=values, skip_cache=False)
+
+        # Retrieve the inserted item from the cache
+        result = self.data.get_cache_item(key='1', cache_name=cache_name)
+
+        # Assert that the data in the cache matches what was inserted
+        self.assertIsNotNone(result, "No data found in the cache for the inserted ID.")
+        self.assertEqual(result.iloc[0]['user_name'], 'Alice', "The name in the cache doesn't match the inserted value.")
+        self.assertEqual(result.iloc[0]['age'], 30, "The age in the cache does not match the inserted value.")
+
+        # Now test with skipping the cache
+        print("Testing insert_row() with skip_cache=True")
+
+        # Insert another row into the database, this time skipping the cache
+        self.data.insert_row(cache_name=cache_name, columns=columns, values=('Bob', 40), skip_cache=True)
+
+        # Attempt to retrieve the newly inserted row from the cache
+        result_after_skip = self.data.get_cache_item(key='2', cache_name=cache_name)
+
+        # Assert that no data is found in the cache for the new row
+        self.assertIsNone(result_after_skip, "Data should not have been cached when skip_cache=True.")
+
+        print(" - Insert row with and without caching passed all checks.")
 
     def test_fill_data_holes(self):
         print('Testing _fill_data_holes() method:')