From ba8cce073e41163ae43e3b2ebb839edfa3c0b46d Mon Sep 17 00:00:00 2001 From: berkay Date: Tue, 25 Mar 2025 10:45:20 +0300 Subject: [PATCH] redis added --- Events/Engine/set_defaults/setClusters.py | 4 +- Services/RedisService/Actions/actions.py | 159 +++++++++++ Services/RedisService/Models/access.py | 36 +++ Services/RedisService/Models/base.py | 309 ++++++++++++++++++++++ Services/RedisService/Models/cluster.py | 17 ++ Services/RedisService/Models/response.py | 68 +++++ Services/RedisService/Models/row.py | 20 ++ Services/RedisService/conn.py | 25 ++ 8 files changed, 636 insertions(+), 2 deletions(-) create mode 100644 Services/RedisService/Actions/actions.py create mode 100644 Services/RedisService/Models/access.py create mode 100644 Services/RedisService/Models/base.py create mode 100644 Services/RedisService/Models/cluster.py create mode 100644 Services/RedisService/Models/response.py create mode 100644 Services/RedisService/Models/row.py create mode 100644 Services/RedisService/conn.py diff --git a/Events/Engine/set_defaults/setClusters.py b/Events/Engine/set_defaults/setClusters.py index 71a5aca..56e01e3 100644 --- a/Events/Engine/set_defaults/setClusters.py +++ b/Events/Engine/set_defaults/setClusters.py @@ -9,8 +9,8 @@ from ApiLayers.AllConfigs.Redis.configs import ( RedisCategoryPageInfoKeys, ) from Events.Engine.abstract_class import CategoryCluster -from Services.Redis.Actions.actions import RedisActions -from Services.Redis.Models.cluster import RedisList +from Services.RedisService.Actions.actions import RedisActions +from Services.RedisService.Models.cluster import RedisList from .prepare_redis_items import DecoratorModule, PrepareRedisItems from .category_cluster_models import CategoryClusterController diff --git a/Services/RedisService/Actions/actions.py b/Services/RedisService/Actions/actions.py new file mode 100644 index 0000000..ec8f506 --- /dev/null +++ b/Services/RedisService/Actions/actions.py @@ -0,0 +1,159 @@ +import arrow + +from typing import Optional, List, Dict, Union + +from ApiLayers.AllConfigs.main import MainConfig + +from Services.RedisService.conn import redis_cli +from Services.RedisService.Models.base import RedisRow +from Services.RedisService.Models.response import RedisResponse + + +class RedisActions: + """Class for handling Redis operations with JSON data.""" + + @classmethod + def get_expiry_time(cls, expiry_kwargs: Dict[str, int]) -> int: + """Calculate expiry time in seconds from kwargs.""" + time_multipliers = {"days": 86400, "hours": 3600, "minutes": 60, "seconds": 1} + return sum( + int(expiry_kwargs.get(unit, 0)) * multiplier + for unit, multiplier in time_multipliers.items() + ) + + @classmethod + def set_expiry_time(cls, expiry_seconds: int) -> Dict[str, int]: + """Convert total seconds back into a dictionary of time units.""" + time_multipliers = {"days": 86400, "hours": 3600, "minutes": 60, "seconds": 1} + result = {} + for unit, multiplier in time_multipliers.items(): + if expiry_seconds >= multiplier: + result[unit], expiry_seconds = divmod(expiry_seconds, multiplier) + return result + + @classmethod + def resolve_expires_at(cls, redis_row: RedisRow) -> str: + """Resolve expiry time for Redis key.""" + expiry_time = redis_cli.ttl(redis_row.redis_key) + if expiry_time == -1: + return "Key has no expiry time." + return arrow.now().shift(seconds=expiry_time).format(MainConfig.DATETIME_FORMAT) + + @classmethod + def delete_key(cls, key: Union[Optional[str], Optional[bytes]]): + try: + redis_cli.delete(key) + return RedisResponse( + status=True, + message="Value is deleted successfully.", + ) + except Exception as e: + return RedisResponse( + status=False, + message="Value is not deleted successfully.", + error=str(e), + ) + + @classmethod + def delete( + cls, list_keys: List[Union[Optional[str], Optional[bytes]]] + ) -> RedisResponse: + try: + regex = RedisRow().regex(list_keys=list_keys) + json_get = redis_cli.scan_iter(match=regex) + + for row in list(json_get): + redis_cli.delete(row) + + return RedisResponse( + status=True, + message="Values are deleted successfully.", + ) + except Exception as e: + return RedisResponse( + status=False, + message="Values are not deleted successfully.", + error=str(e), + ) + + @classmethod + def set_json( + cls, + list_keys: List[Union[str, bytes]], + value: Optional[Union[Dict, List]], + expires: Optional[Dict[str, int]] = None, + ) -> RedisResponse: + """Set JSON value in Redis with optional expiry.""" + redis_row = RedisRow() + redis_row.merge(set_values=list_keys) + redis_row.feed(value) + redis_row.expires_at_string = None + redis_row.expires_at = None + try: + if expires: + redis_row.expires_at = expires + expiry_time = cls.get_expiry_time(expiry_kwargs=expires) + redis_cli.setex( + name=redis_row.redis_key, + time=expiry_time, + value=redis_row.value, + ) + redis_row.expires_at_string = str( + arrow.now() + .shift(seconds=expiry_time) + .format(MainConfig.DATETIME_FORMAT) + ) + else: + redis_cli.set(name=redis_row.redis_key, value=redis_row.value) + + return RedisResponse( + status=True, + message="Value is set successfully.", + data=redis_row, + ) + except Exception as e: + return RedisResponse( + status=False, + message="Value is not set successfully.", + error=str(e), + ) + + @classmethod + def get_json( + cls, list_keys: List[Union[Optional[str], Optional[bytes]]] + ) -> RedisResponse: + """Get JSON values from Redis using pattern matching.""" + try: + list_of_rows = [] + regex = RedisRow.regex(list_keys=list_keys) + json_get = redis_cli.scan_iter(match=regex) + for row in list(json_get): + redis_row = RedisRow() + redis_row.set_key(key=row) + redis_value = redis_cli.get(row) + redis_value_expire = redis_cli.ttl(row) + redis_row.expires_at = cls.set_expiry_time( + expiry_seconds=int(redis_value_expire) + ) + redis_row.expires_at_string = cls.resolve_expires_at( + redis_row=redis_row + ) + redis_row.feed(redis_value) + list_of_rows.append(redis_row) + if list_of_rows: + return RedisResponse( + status=True, + message="Value is get successfully.", + data=list_of_rows, + ) + return RedisResponse( + status=False, + message="Value is not get successfully.", + data=list_of_rows, + ) + except Exception as e: + return RedisResponse( + status=False, + message="Value is not get successfully.", + error=str(e), + ) diff --git a/Services/RedisService/Models/access.py b/Services/RedisService/Models/access.py new file mode 100644 index 0000000..d060041 --- /dev/null +++ b/Services/RedisService/Models/access.py @@ -0,0 +1,36 @@ +from typing import Optional +from uuid import UUID +from pydantic import field_validator + +from ApiLayers.AllConfigs.Redis.configs import RedisAuthKeys +from Services.RedisService.Models.row import BaseRedisModel + + +class AccessToken(BaseRedisModel): + + auth_key: Optional[str] = RedisAuthKeys.AUTH + accessToken: Optional[str] = None + userUUID: Optional[str | UUID] = None + + @field_validator("userUUID", mode="after") + def validate_uuid(cls, v): + """Convert UUID to string during validation.""" + if v is None: + return None + return str(v) + + def to_list(self): + """Convert to list for Redis storage.""" + return [ + self.auth_key, + self.accessToken, + str(self.userUUID) if self.userUUID else None, + ] + + @property + def count(self): + return 3 + + @property + def delimiter(self): + return ":" diff --git a/Services/RedisService/Models/base.py b/Services/RedisService/Models/base.py new file mode 100644 index 0000000..253d443 --- /dev/null +++ b/Services/RedisService/Models/base.py @@ -0,0 +1,309 @@ +""" +Redis key-value operations with structured data handling. + +This module provides a class for managing Redis key-value operations with support for: +- Structured data storage and retrieval +- Key pattern generation for searches +- JSON serialization/deserialization +- Type-safe value handling +""" + +import arrow +import json +from typing import Union, Dict, List, Optional, Any, ClassVar +from Services.RedisService.conn import redis_cli + + +class RedisKeyError(Exception): + """Exception raised for Redis key-related errors.""" + + pass + + +class RedisValueError(Exception): + """Exception raised for Redis value-related errors.""" + + pass + + +class RedisRow: + """ + Handles Redis key-value operations with structured data. + + This class provides methods for: + - Managing compound keys with delimiters + - Converting between bytes and string formats + - JSON serialization/deserialization of values + - Pattern generation for Redis key searches + + Attributes: + key: The Redis key in bytes or string format + value: The stored value (will be JSON serialized) + delimiter: Character used to separate compound key parts + expires_at: Optional expiration timestamp + """ + + key: ClassVar[Union[str, bytes]] + value: ClassVar[Any] + delimiter: str = ":" + expires_at: Optional[dict] = {"seconds": 60 * 60 * 30} + expires_at_string: Optional[str] + + def get_expiry_time(self) -> int | None: + """Calculate expiry time in seconds from kwargs.""" + time_multipliers = {"days": 86400, "hours": 3600, "minutes": 60, "seconds": 1} + if self.expires_at: + return sum( + int(self.expires_at.get(unit, 0)) * multiplier + for unit, multiplier in time_multipliers.items() + ) + return + + def merge(self, set_values: List[Union[str, bytes]]) -> None: + """ + Merge list of values into a single delimited key. + + Args: + set_values: List of values to merge into key + + Example: + >>> RedisRow.merge(["users", "123", "profile"]) + >>> print(RedisRow.key) + b'users:123:profile' + """ + if not set_values: + raise RedisKeyError("Cannot merge empty list of values") + + merged = [] + for value in set_values: + if value is None: + continue + if isinstance(value, bytes): + value = value.decode() + merged.append(str(value)) + + self.key = self.delimiter.join(merged).encode() + + @classmethod + def regex(cls, list_keys: List[Union[str, bytes, None]]) -> str: + """ + Generate Redis search pattern from list of keys. + + Args: + list_keys: List of key parts, can include None for wildcards + + Returns: + str: Redis key pattern with wildcards + + Example: + >>> RedisRow.regex([None, "users", "active"]) + '*:users:active' + """ + if not list_keys: + return "" + + # Filter and convert valid keys + valid_keys = [] + for key in list_keys: + if key is None or str(key) == "None": + continue + if isinstance(key, bytes): + key = key.decode() + valid_keys.append(str(key)) + + # Build pattern + pattern = cls.delimiter.join(valid_keys) + if not pattern: + return "" + + # Add wildcard if first key was None + if list_keys[0] is None: + pattern = f"*{cls.delimiter}{pattern}" + if "*" not in pattern and any([list_key is None for list_key in list_keys]): + pattern = f"{pattern}:*" + return pattern + + def parse(self) -> List[str]: + """ + Parse the key into its component parts. + + Returns: + List[str]: Key parts split by delimiter + + Example: + >>> RedisRow.key = b'users:123:profile' + >>> RedisRow.parse() + ['users', '123', 'profile'] + """ + if not self.key: + return [] + + key_str = self.key.decode() if isinstance(self.key, bytes) else self.key + return key_str.split(self.delimiter) + + def feed(self, value: Union[bytes, Dict, List, str]) -> None: + """ + Convert and store value in JSON format. + + Args: + value: Value to store (bytes, dict, or list) + + Raises: + RedisValueError: If value type is not supported + + Example: + >>> RedisRow.feed({"name": "John", "age": 30}) + >>> print(RedisRow.value) + '{"name": "John", "age": 30}' + """ + try: + if isinstance(value, (dict, list)): + self.value = json.dumps(value) + elif isinstance(value, bytes): + self.value = json.dumps(json.loads(value.decode())) + elif isinstance(value, str): + self.value = value + else: + raise RedisValueError(f"Unsupported value type: {type(value)}") + except json.JSONDecodeError as e: + raise RedisValueError(f"Invalid JSON format: {str(e)}") + + def modify(self, add_dict: Dict) -> None: + """ + Modify existing data by merging with new dictionary. + + Args: + add_dict: Dictionary to merge with existing data + + Example: + >>> RedisRow.feed({"name": "John"}) + >>> RedisRow.modify({"age": 30}) + >>> print(RedisRow.data) + {"name": "John", "age": 30} + """ + if not isinstance(add_dict, dict): + raise RedisValueError("modify() requires a dictionary argument") + current_data = self.row if self.row else {} + if not isinstance(current_data, dict): + raise RedisValueError("Cannot modify non-dictionary data") + current_data = { + **current_data, + **add_dict, + } + self.feed(current_data) + self.save() + + def save(self): + """ + Save the data to Redis with optional expiration. + + Raises: + RedisKeyError: If key is not set + RedisValueError: If value is not set + """ + + if not self.key: + raise RedisKeyError("Cannot save data without a key") + if not self.value: + raise RedisValueError("Cannot save empty data") + + if self.expires_at: + redis_cli.setex( + name=self.redis_key, time=self.get_expiry_time(), value=self.value + ) + self.expires_at_string = str( + arrow.now() + .shift(seconds=self.get_expiry_time()) + .format("YYYY-MM-DD HH:mm:ss") + ) + return self.value + redis_cli.set(name=self.redis_key, value=self.value) + self.expires_at = None + self.expires_at_string = None + return self.value + + def remove(self, key: str) -> None: + """ + Remove a key from the stored dictionary. + + Args: + key: Key to remove from stored dictionary + + Raises: + KeyError: If key doesn't exist + RedisValueError: If stored value is not a dictionary + """ + current_data = self.row + if not isinstance(current_data, dict): + raise RedisValueError("Cannot remove key from non-dictionary data") + + try: + current_data.pop(key) + self.feed(current_data) + self.save() + except KeyError: + raise KeyError(f"Key '{key}' not found in stored data") + + def delete(self) -> None: + """Delete the key from Redis.""" + try: + redis_cli.delete(self.redis_key) + except Exception as e: + print(f"Error deleting key: {str(e)}") + + @property + def keys(self) -> str: + """ + Get key as string. + + Returns: + str: Key in string format + """ + return self.key.decode() if isinstance(self.key, bytes) else self.key + + def set_key(self, key: Union[str, bytes]) -> None: + """ + Set key ensuring bytes format. + + Args: + key: Key in string or bytes format + """ + if not key: + raise RedisKeyError("Cannot set empty key") + self.key = key if isinstance(key, bytes) else str(key).encode() + + @property + def redis_key(self) -> bytes: + """ + Get key in bytes format for Redis operations. + + Returns: + bytes: Key in bytes format + """ + return self.key if isinstance(self.key, bytes) else str(self.key).encode() + + @property + def row(self) -> Union[Dict, List]: + """ + Get stored value as Python object. + + Returns: + Union[Dict, List]: Deserialized JSON data + """ + try: + return json.loads(self.value) + except json.JSONDecodeError as e: + raise RedisValueError(f"Invalid JSON format in stored value: {str(e)}") + + @property + def as_dict(self) -> Dict[str, Any]: + """ + Get row data as dictionary. + + Returns: + Dict[str, Any]: Dictionary with keys and value + """ + return { + "keys": self.keys, + "value": self.row, + } diff --git a/Services/RedisService/Models/cluster.py b/Services/RedisService/Models/cluster.py new file mode 100644 index 0000000..dd06bc0 --- /dev/null +++ b/Services/RedisService/Models/cluster.py @@ -0,0 +1,17 @@ +from Services.RedisService.Models.row import BaseRedisModel + + +class RedisList(BaseRedisModel): + redis_key: str + + def to_list(self): + """Convert to list for Redis storage.""" + return [self.redis_key] + + @property + def count(self): + return 1 + + @property + def delimiter(self): + return ":" diff --git a/Services/RedisService/Models/response.py b/Services/RedisService/Models/response.py new file mode 100644 index 0000000..68cf3a9 --- /dev/null +++ b/Services/RedisService/Models/response.py @@ -0,0 +1,68 @@ +from typing import Union, Dict, List, Optional, Any +from Services.RedisService.Models.base import RedisRow + + +class RedisResponse: + """Base class for Redis response handling.""" + + def __init__( + self, + status: bool, + message: str, + data: Any = None, + error: Optional[str] = None, + ): + self.status = status + self.message = message + self.data = data + + if isinstance(data, dict): + self.data_type = "dict" + elif isinstance(data, list): + self.data_type = "list" + elif isinstance(data, RedisRow): + self.data_type = "row" + elif data is None: + self.data_type = None + self.error = error + + def as_dict(self) -> Dict: + data = self.all + main_dict = { + "status": self.status, + "message": self.message, + "count": self.count, + "dataType": getattr(self, "data_type", None), + } + if isinstance(data, RedisRow): + dict_return = {data.keys: data.row} + dict_return.update(dict(main_dict)) + return dict_return + elif isinstance(data, list): + dict_return = {row.keys: row.data for row in data} + dict_return.update(dict(main_dict)) + return dict_return + + @property + def all(self) -> Union[Optional[List[RedisRow]]]: + return self.data or [] + + @property + def count(self) -> int: + row = self.all + if isinstance(row, list): + return len(row) + elif isinstance(row, RedisRow): + return 1 + + @property + def first(self) -> Union[RedisRow, dict, None]: + if self.data: + if isinstance(self.data, list): + if isinstance(self.data[0], RedisRow): + return self.data[0].row + return self.data[0] + elif isinstance(self.data, RedisRow): + return self.data.row + self.status = False + return diff --git a/Services/RedisService/Models/row.py b/Services/RedisService/Models/row.py new file mode 100644 index 0000000..b9fd12b --- /dev/null +++ b/Services/RedisService/Models/row.py @@ -0,0 +1,20 @@ +from abc import abstractmethod +from pydantic import BaseModel + + +class BaseRedisModel(BaseModel): + + @abstractmethod + def to_list(self) -> list: + """Convert to list for Redis storage.""" + pass + + @abstractmethod + def count(self) -> int: + """Return the number of elements in the list.""" + pass + + @abstractmethod + def delimiter(self) -> str: + """Return the delimiter for the list.""" + pass diff --git a/Services/RedisService/conn.py b/Services/RedisService/conn.py new file mode 100644 index 0000000..d9e7d0d --- /dev/null +++ b/Services/RedisService/conn.py @@ -0,0 +1,25 @@ +from redis import Redis + +from ApiLayers.AllConfigs.Redis.configs import WagRedis + + +class RedisConn: + + def __init__(self): + self.redis = Redis(**WagRedis.as_dict()) + if not self.check_connection(): + raise Exception("Connection error") + + def check_connection(self): + return self.redis.ping() + + def set_connection(self, host, password, port, db): + self.redis = Redis(host=host, password=password, port=port, db=db) + return self.redis + + +try: + redis_conn = RedisConn() + redis_cli = redis_conn.redis +except Exception as e: + print("Redis Connection Error", e)