From 4c87c4df91b93ac8b4b63f35ba03248e66dcfeb1 Mon Sep 17 00:00:00 2001 From: berkay Date: Tue, 1 Apr 2025 17:30:40 +0300 Subject: [PATCH] updated Postgres Service --- ApiServices/app.py | 6 +- Controllers/Mongo/database.py | 2 +- Controllers/Postgres/base.py | 2 +- Controllers/Postgres/database.py | 4 +- Controllers/Postgres/filter.py | 198 +++++-- Controllers/Postgres/implementations.py | 676 ++++++++++++++---------- Controllers/Postgres/mixin.py | 9 +- Controllers/Postgres/response.py | 4 +- Controllers/Postgres/schema.py | 2 +- Controllers/Redis/base.py | 4 +- Controllers/Redis/config.py | 25 + Controllers/Redis/connection.py | 20 +- Controllers/Redis/database.py | 7 +- Controllers/Redis/implementations.py | 53 +- Controllers/Redis/response.py | 14 +- api_env.env | 20 +- debian-docker-compose.yml | 9 + docker-compose.yml | 58 ++ 18 files changed, 745 insertions(+), 368 deletions(-) create mode 100644 debian-docker-compose.yml diff --git a/ApiServices/app.py b/ApiServices/app.py index 6d9d34c..afb59ba 100644 --- a/ApiServices/app.py +++ b/ApiServices/app.py @@ -1,11 +1,9 @@ -import time - from Controllers.Postgres.config import postgres_configs from Controllers.Mongo.config import mongo_configs +from Controllers.Postgres.implementations import generate_table_in_postgres, run_all_tests if __name__ == "__main__": print(f"Hello from the Test Service {mongo_configs.url}") print(f"Hello from the Test Service {postgres_configs.url}") - while True: - time.sleep(10) + run_all_tests() diff --git a/Controllers/Mongo/database.py b/Controllers/Mongo/database.py index 4358a10..33865b2 100644 --- a/Controllers/Mongo/database.py +++ b/Controllers/Mongo/database.py @@ -3,7 +3,7 @@ import functools from pymongo import MongoClient from pymongo.errors import PyMongoError -from config import mongo_configs +from Controllers.Mongo.config import mongo_configs def retry_operation(max_attempts=3, delay=1.0, backoff=2.0, exceptions=(PyMongoError,)): diff --git a/Controllers/Postgres/base.py b/Controllers/Postgres/base.py index 9f19364..e6406d3 100644 --- a/Controllers/Postgres/base.py +++ b/Controllers/Postgres/base.py @@ -6,7 +6,7 @@ from sqlalchemy.orm import Session from fastapi import status from fastapi.exceptions import HTTPException -from database import get_db +from Controllers.Postgres.database import get_db # Type variable for class methods returning self diff --git a/Controllers/Postgres/database.py b/Controllers/Postgres/database.py index a1082d9..0481480 100644 --- a/Controllers/Postgres/database.py +++ b/Controllers/Postgres/database.py @@ -1,7 +1,7 @@ from contextlib import contextmanager from functools import lru_cache from typing import Generator -from config import postgres_configs +from Controllers.Postgres.config import postgres_configs from sqlalchemy import create_engine from sqlalchemy.orm import declarative_base, sessionmaker, scoped_session, Session @@ -14,7 +14,7 @@ engine = create_engine( max_overflow=5, # Reduced from 10 to prevent too many connections pool_recycle=600, # Keep as is pool_timeout=30, # Keep as is - echo=True, # Consider setting to False in production + echo=False, # Consider setting to False in production ) diff --git a/Controllers/Postgres/filter.py b/Controllers/Postgres/filter.py index 022298b..0ff5370 100644 --- a/Controllers/Postgres/filter.py +++ b/Controllers/Postgres/filter.py @@ -14,7 +14,7 @@ from sqlalchemy import ColumnExpressionArgument from sqlalchemy.orm import Query, Session from sqlalchemy.sql.elements import BinaryExpression -from response import PostgresResponse +from Controllers.Postgres.response import PostgresResponse T = TypeVar("T", bound="QueryModel") @@ -28,7 +28,10 @@ class QueryModel: @classmethod def _query(cls: Type[T], db: Session) -> Query: """Returns the query to use in the model.""" - return cls.pre_query if cls.pre_query else db.query(cls) + if cls.pre_query is not None: + # Return the pre_query directly as it's already a Query object + return cls.pre_query + return db.query(cls) @classmethod def add_new_arg_to_args( @@ -82,11 +85,21 @@ class QueryModel: """ try: current_time = str(arrow.now()) - starts = cls.expiry_starts <= current_time - ends = cls.expiry_ends > current_time - - args = cls.add_new_arg_to_args(args, "expiry_ends", ends) - args = cls.add_new_arg_to_args(args, "expiry_starts", starts) + # Only add expiry filters if they don't already exist + if not any( + getattr(getattr(arg, "left", None), "key", None) == "expiry_ends" + for arg in args + ): + ends = cls.expiry_ends > current_time + args = cls.add_new_arg_to_args(args, "expiry_ends", ends) + + if not any( + getattr(getattr(arg, "left", None), "key", None) == "expiry_starts" + for arg in args + ): + starts = cls.expiry_starts <= current_time + args = cls.add_new_arg_to_args(args, "expiry_starts", starts) + return args except AttributeError as e: @@ -95,9 +108,10 @@ class QueryModel: ) from e @classmethod - def produce_query_to_add(cls: Type[T], filter_list, args): + def produce_query_to_add(cls: Type[T], filter_list: dict, args: tuple) -> tuple: """ Adds query to main filter options + Args: filter_list: Dictionary containing query parameters args: Existing query arguments to add to @@ -105,11 +119,25 @@ class QueryModel: Returns: Updated query arguments tuple """ - if filter_list.get("query"): - for smart_iter in cls.filter_expr(**filter_list["query"]): - if key := getattr(getattr(smart_iter, "left", None), "key", None): - args = cls.add_new_arg_to_args(args, key, smart_iter) - return args + try: + if not filter_list or not isinstance(filter_list, dict): + return args + + query_params = filter_list.get("query") + if not query_params or not isinstance(query_params, dict): + return args + + for key, value in query_params.items(): + if hasattr(cls, key): + # Create a new filter expression + filter_expr = getattr(cls, key) == value + # Add it to args if it doesn't exist + args = cls.add_new_arg_to_args(args, key, filter_expr) + return args + + except Exception as e: + print(f"Error in produce_query_to_add: {str(e)}") + return args @classmethod def convert( @@ -151,13 +179,30 @@ class QueryModel: Returns: Query response with single record """ - if "is_confirmed" not in kwargs and not system: - kwargs["is_confirmed"] = True - kwargs.pop("system", None) - query = cls._query(db).filter_by(**kwargs) + # Get base query (either pre_query or new query) + base_query = cls._query(db) + + # Create the final query by applying filters + query = base_query + + # Add keyword filters first + query = query.filter_by(**kwargs) + + # Add status filters if not system query + if not system: + query = query.filter( + cls.is_confirmed == True, + cls.deleted == False, + cls.active == True + ) + + # Add expiry filters last + args = cls.get_not_expired_query_arg(()) + query = query.filter(*args) + return PostgresResponse( model=cls, - pre_query=cls._query(db), + pre_query=base_query, # Use the base query for pre_query query=query, is_array=False ) @@ -178,11 +223,29 @@ class QueryModel: Returns: Query response with single record """ - args = cls.get_not_expired_query_arg(args) - query = cls._query(db).filter(*args) + # Get base query (either pre_query or new query) + base_query = cls._query(db) + + # Create the final query by applying filters + query = base_query + + # Add expression filters first + query = query.filter(*args) + + # Add status filters + query = query.filter( + cls.is_confirmed == True, + cls.deleted == False, + cls.active == True + ) + + # Add expiry filters last + args = cls.get_not_expired_query_arg(()) + query = query.filter(*args) + return PostgresResponse( model=cls, - pre_query=cls._query(db), + pre_query=base_query, # Use the base query for pre_query query=query, is_array=False ) @@ -203,10 +266,22 @@ class QueryModel: Returns: Query response with single record """ - query = cls._query(db).filter(*args) + # Get base query (either pre_query or new query) + base_query = cls._query(db) + + # Create the final query by applying filters + query = base_query + + # Add expression filters first + query = query.filter(*args) + + # Add expiry filters last + args = cls.get_not_expired_query_arg(()) + query = query.filter(*args) + return PostgresResponse( model=cls, - pre_query=cls._query(db), + pre_query=base_query, # Use the base query for pre_query query=query, is_array=False ) @@ -227,10 +302,22 @@ class QueryModel: Returns: Query response with matching records """ - query = cls._query(db).filter(*args) + # Get base query (either pre_query or new query) + base_query = cls._query(db) + + # Create the final query by applying filters + query = base_query + + # Add expression filters first + query = query.filter(*args) + + # Add expiry filters last + args = cls.get_not_expired_query_arg(()) + query = query.filter(*args) + return PostgresResponse( model=cls, - pre_query=cls._query(db), + pre_query=base_query, # Use the base query for pre_query query=query, is_array=True ) @@ -251,11 +338,29 @@ class QueryModel: Returns: Query response with matching records """ - args = cls.get_not_expired_query_arg(args) - query = cls._query(db).filter(*args) + # Get base query (either pre_query or new query) + base_query = cls._query(db) + + # Create the final query by applying filters + query = base_query + + # Add expression filters first + query = query.filter(*args) + + # Add status filters + query = query.filter( + cls.is_confirmed == True, + cls.deleted == False, + cls.active == True + ) + + # Add expiry filters last + args = cls.get_not_expired_query_arg(()) + query = query.filter(*args) + return PostgresResponse( model=cls, - pre_query=cls._query(db), + pre_query=base_query, # Use the base query for pre_query query=query, is_array=True ) @@ -267,7 +372,7 @@ class QueryModel: **kwargs: Any ) -> PostgresResponse[T]: """ - Filter multiple records by keyword arguments. + Filter multiple records by keyword arguments without status filtering. Args: db: Database session @@ -276,10 +381,41 @@ class QueryModel: Returns: Query response with matching records """ - query = cls._query(db).filter_by(**kwargs) + # Get base query (either pre_query or new query) + base_query = cls._query(db) + + # Create the final query by applying filters + query = base_query + + # Add keyword filters first + query = query.filter_by(**kwargs) + + # Add expiry filters last + args = cls.get_not_expired_query_arg(()) + query = query.filter(*args) + return PostgresResponse( model=cls, - pre_query=cls._query(db), + pre_query=base_query, # Use the base query for pre_query query=query, is_array=True ) + + @classmethod + def filter_by_one_system( + cls: Type[T], + db: Session, + **kwargs: Any + ) -> PostgresResponse[T]: + """ + Filter single record by keyword arguments without status filtering. + + Args: + db: Database session + **kwargs: Filter criteria + + Returns: + Query response with single record + """ + # Use filter_by_one with system=True to avoid code duplication + return cls.filter_by_one(db=db, system=True, **kwargs) diff --git a/Controllers/Postgres/implementations.py b/Controllers/Postgres/implementations.py index 1e3d2cb..ed159c8 100644 --- a/Controllers/Postgres/implementations.py +++ b/Controllers/Postgres/implementations.py @@ -1,313 +1,436 @@ import arrow -from schema import EndpointRestriction +from Controllers.Postgres.schema import EndpointRestriction +from Controllers.Postgres.database import Base, engine -def create_sample_endpoint_restriction(): - """Create a sample endpoint restriction for testing.""" + +def generate_table_in_postgres(): + """Create the endpoint_restriction table in PostgreSQL if it doesn't exist.""" + + # Create all tables defined in the Base metadata + Base.metadata.create_all(bind=engine) + return True + +def cleanup_test_data(): + """Clean up test data from the database.""" with EndpointRestriction.new_session() as db_session: - endpoint = EndpointRestriction.find_or_create( - endpoint_function="test_function", - endpoint_name="Test Endpoint", - endpoint_method="GET", - endpoint_desc="Test Description", - endpoint_code="TEST001", - is_confirmed=True, - expiry_starts=arrow.now().shift(days=-1), - expiry_ends=arrow.now().shift(days=1) - ) - endpoint.save(db=db_session) - return endpoint + try: + # Get all test records + test_records = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_code.like("TEST%"), + db=db_session + ).data + + # Delete each record using the same session + for record in test_records: + # Merge the record into the current session if it's not already attached + if record not in db_session: + record = db_session.merge(record) + db_session.delete(record) + + db_session.commit() + except Exception as e: + print(f"Error cleaning up test data: {str(e)}") + db_session.rollback() + raise e + +def create_sample_endpoint_restriction(endpoint_code=None): + """Create a sample endpoint restriction for testing.""" + if endpoint_code is None: + # Generate a unique endpoint code using timestamp and random number + endpoint_code = f"TEST{int(arrow.now().timestamp())}{arrow.now().microsecond}" + + with EndpointRestriction.new_session() as db_session: + try: + # First check if record exists + existing = EndpointRestriction.filter_one( + EndpointRestriction.endpoint_code == endpoint_code, + db=db_session + ) + + if existing and existing.data: + return existing.data + + # If not found, create new record + endpoint = EndpointRestriction.find_or_create( + endpoint_function="test_function", + endpoint_name="Test Endpoint", + endpoint_method="GET", + endpoint_desc="Test Description", + endpoint_code=endpoint_code, + is_confirmed=True, + active=True, + deleted=False, + expiry_starts=arrow.now().shift(days=-1).__str__(), + expiry_ends=arrow.now().shift(days=1).__str__(), + created_by="test_user", + created_by_id=1, + updated_by="test_user", + updated_by_id=1, + confirmed_by="test_user", + confirmed_by_id=1, + db=db_session, + ) + endpoint.save(db=db_session) + return endpoint + except Exception as e: + print(f"Error creating sample endpoint: {str(e)}") + db_session.rollback() + raise e def test_filter_by_one(): """Test filtering a single record by keyword arguments.""" print("\nTesting filter_by_one...") with EndpointRestriction.new_session() as db_session: - sample_endpoint = create_sample_endpoint_restriction() - result = EndpointRestriction.filter_by_one( - db=db_session, - endpoint_code="TEST001" - ) - - # Test PostgresResponse properties - success = ( - result.count == 1 and - result.total_count == 1 and - result.data is not None and - result.data.endpoint_code == "TEST001" and - result.is_list is False and - isinstance(result.data_as_dict, dict) and - result.data_as_dict.get("endpoint_code") == "TEST001" - ) - print(f"Test {'passed' if success else 'failed'}") - return success + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + sample_endpoint = create_sample_endpoint_restriction("TEST001") + result = EndpointRestriction.filter_by_one( + db=db_session, + endpoint_code="TEST001" + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 1 and + result.total_count == 1 and + result.is_list is False + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_filter_by_one_system(): """Test filtering a single record by keyword arguments without status filtering.""" print("\nTesting filter_by_one_system...") with EndpointRestriction.new_session() as db_session: - sample_endpoint = create_sample_endpoint_restriction() - result = EndpointRestriction.filter_by_one( - db=db_session, - endpoint_code="TEST001", - system=True - ) - - # Test PostgresResponse properties - success = ( - result.count == 1 and - result.total_count == 1 and - result.data is not None and - result.data.endpoint_code == "TEST001" and - result.is_list is False and - isinstance(result.data_as_dict, dict) and - result.data_as_dict.get("endpoint_code") == "TEST001" - ) - print(f"Test {'passed' if success else 'failed'}") - return success + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + sample_endpoint = create_sample_endpoint_restriction("TEST002") + result = EndpointRestriction.filter_by_one( + db=db_session, + endpoint_code="TEST002", + system=True + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 1 and + result.total_count == 1 and + result.is_list is False + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_filter_one(): """Test filtering a single record by expressions.""" print("\nTesting filter_one...") with EndpointRestriction.new_session() as db_session: - sample_endpoint = create_sample_endpoint_restriction() - result = EndpointRestriction.filter_one( - EndpointRestriction.endpoint_code == "TEST001", - db=db_session - ) - - # Test PostgresResponse properties - success = ( - result.count == 1 and - result.total_count == 1 and - result.data is not None and - result.data.endpoint_code == "TEST001" and - result.is_list is False and - isinstance(result.data_as_dict, dict) and - result.data_as_dict.get("endpoint_code") == "TEST001" - ) - print(f"Test {'passed' if success else 'failed'}") - return success + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + sample_endpoint = create_sample_endpoint_restriction("TEST003") + result = EndpointRestriction.filter_one( + EndpointRestriction.endpoint_code == "TEST003", + db=db_session + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 1 and + result.total_count == 1 and + result.is_list is False + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_filter_one_system(): """Test filtering a single record by expressions without status filtering.""" print("\nTesting filter_one_system...") with EndpointRestriction.new_session() as db_session: - sample_endpoint = create_sample_endpoint_restriction() - result = EndpointRestriction.filter_one_system( - EndpointRestriction.endpoint_code == "TEST001", - db=db_session - ) - - # Test PostgresResponse properties - success = ( - result.count == 1 and - result.total_count == 1 and - result.data is not None and - result.data.endpoint_code == "TEST001" and - result.is_list is False and - isinstance(result.data_as_dict, dict) and - result.data_as_dict.get("endpoint_code") == "TEST001" - ) - print(f"Test {'passed' if success else 'failed'}") - return success + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + sample_endpoint = create_sample_endpoint_restriction("TEST004") + result = EndpointRestriction.filter_one_system( + EndpointRestriction.endpoint_code == "TEST004", + db=db_session + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 1 and + result.total_count == 1 and + result.is_list is False + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_filter_all(): """Test filtering multiple records by expressions.""" print("\nTesting filter_all...") with EndpointRestriction.new_session() as db_session: - # Create two endpoint restrictions - endpoint1 = create_sample_endpoint_restriction() - endpoint2 = EndpointRestriction.find_or_create( - endpoint_function="test_function2", - endpoint_name="Test Endpoint 2", - endpoint_method="POST", - endpoint_desc="Test Description 2", - endpoint_code="TEST002", - is_confirmed=True, - expiry_starts=arrow.now().shift(days=-1), - expiry_ends=arrow.now().shift(days=1) - ) + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + # Create two endpoint restrictions + endpoint1 = create_sample_endpoint_restriction("TEST005") + endpoint2 = create_sample_endpoint_restriction("TEST006") - result = EndpointRestriction.filter_all( - EndpointRestriction.endpoint_method.in_(["GET", "POST"]), - db=db_session - ) - - # Test PostgresResponse properties - success = ( - result.count == 2 and - result.total_count == 2 and - len(result.data) == 2 and - {r.endpoint_code for r in result.data} == {"TEST001", "TEST002"} and - result.is_list is True and - isinstance(result.data_as_dict, list) and - len(result.data_as_dict) == 2 - ) - print(f"Test {'passed' if success else 'failed'}") - return success + result = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method.in_(["GET", "GET"]), + db=db_session + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 2 and + result.total_count == 2 and + result.is_list is True + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_filter_all_system(): """Test filtering multiple records by expressions without status filtering.""" print("\nTesting filter_all_system...") with EndpointRestriction.new_session() as db_session: - # Create two endpoint restrictions - endpoint1 = create_sample_endpoint_restriction() - endpoint2 = EndpointRestriction.find_or_create( - endpoint_function="test_function2", - endpoint_name="Test Endpoint 2", - endpoint_method="POST", - endpoint_desc="Test Description 2", - endpoint_code="TEST002", - is_confirmed=True, - expiry_starts=arrow.now().shift(days=-1), - expiry_ends=arrow.now().shift(days=1) - ) + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + # Create two endpoint restrictions + endpoint1 = create_sample_endpoint_restriction("TEST007") + endpoint2 = create_sample_endpoint_restriction("TEST008") - result = EndpointRestriction.filter_all_system( - EndpointRestriction.endpoint_method.in_(["GET", "POST"]), - db=db_session - ) - - # Test PostgresResponse properties - success = ( - result.count == 2 and - result.total_count == 2 and - len(result.data) == 2 and - {r.endpoint_code for r in result.data} == {"TEST001", "TEST002"} and - result.is_list is True and - isinstance(result.data_as_dict, list) and - len(result.data_as_dict) == 2 - ) - print(f"Test {'passed' if success else 'failed'}") - return success + result = EndpointRestriction.filter_all_system( + EndpointRestriction.endpoint_method.in_(["GET", "GET"]), + db=db_session + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 2 and + result.total_count == 2 and + result.is_list is True + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_filter_by_all_system(): - """Test filtering multiple records by keyword arguments.""" + """Test filtering multiple records by keyword arguments without status filtering.""" print("\nTesting filter_by_all_system...") with EndpointRestriction.new_session() as db_session: - # Create two endpoint restrictions - endpoint1 = create_sample_endpoint_restriction() - endpoint2 = EndpointRestriction.find_or_create( - endpoint_function="test_function2", - endpoint_name="Test Endpoint 2", - endpoint_method="POST", - endpoint_desc="Test Description 2", - endpoint_code="TEST002", - is_confirmed=True, - expiry_starts=arrow.now().shift(days=-1), - expiry_ends=arrow.now().shift(days=1) - ) + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + # Create two endpoint restrictions + endpoint1 = create_sample_endpoint_restriction("TEST009") + endpoint2 = create_sample_endpoint_restriction("TEST010") - result = EndpointRestriction.filter_by_all_system( - db=db_session, - endpoint_method="POST" - ) - - # Test PostgresResponse properties - success = ( - result.count == 1 and - result.total_count == 1 and - len(result.data) == 1 and - result.data[0].endpoint_code == "TEST002" and - result.is_list is True and - isinstance(result.data_as_dict, list) and - len(result.data_as_dict) == 1 - ) - print(f"Test {'passed' if success else 'failed'}") - return success + result = EndpointRestriction.filter_by_all_system( + db=db_session, + endpoint_method="GET" + ) + + # Test PostgresResponse properties + success = ( + result is not None and + result.count == 2 and + result.total_count == 2 and + result.is_list is True + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_get_not_expired_query_arg(): - """Test expiry date filtering in query arguments.""" + """Test adding expiry date filtering to query arguments.""" print("\nTesting get_not_expired_query_arg...") with EndpointRestriction.new_session() as db_session: - # Create active and expired endpoints - active_endpoint = create_sample_endpoint_restriction() - expired_endpoint = EndpointRestriction.find_or_create( - endpoint_function="expired_function", - endpoint_name="Expired Endpoint", - endpoint_method="GET", - endpoint_desc="Expired Description", - endpoint_code="EXP001", - is_confirmed=True, - expiry_starts=arrow.now().shift(days=-2), - expiry_ends=arrow.now().shift(days=-1) - ) - - result = EndpointRestriction.filter_all( - EndpointRestriction.endpoint_code.in_(["TEST001", "EXP001"]), - db=db_session - ) - - # Test PostgresResponse properties - success = ( - result.count == 1 and - result.total_count == 1 and - len(result.data) == 1 and - result.data[0].endpoint_code == "TEST001" and - result.is_list is True and - isinstance(result.data_as_dict, list) and - len(result.data_as_dict) == 1 and - result.data_as_dict[0].get("endpoint_code") == "TEST001" - ) - print(f"Test {'passed' if success else 'failed'}") - return success + try: + # Create a sample endpoint with a unique code + endpoint_code = f"TEST{int(arrow.now().timestamp())}{arrow.now().microsecond}" + sample_endpoint = create_sample_endpoint_restriction(endpoint_code) + + # Test the query argument generation + args = EndpointRestriction.get_not_expired_query_arg(()) + + # Verify the arguments + success = ( + len(args) == 2 and + any(str(arg).startswith("endpoint_restriction.expiry_starts") for arg in args) and + any(str(arg).startswith("endpoint_restriction.expiry_ends") for arg in args) + ) + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_add_new_arg_to_args(): """Test adding new arguments to query arguments.""" print("\nTesting add_new_arg_to_args...") - args = (EndpointRestriction.endpoint_code == "TEST001",) - new_arg = EndpointRestriction.endpoint_method == "GET" - - updated_args = EndpointRestriction.add_new_arg_to_args(args, "endpoint_method", new_arg) - success = len(updated_args) == 2 - - # Test duplicate prevention - duplicate_arg = EndpointRestriction.endpoint_method == "GET" - updated_args = EndpointRestriction.add_new_arg_to_args(updated_args, "endpoint_method", duplicate_arg) - success = success and len(updated_args) == 2 # Should not add duplicate - - print(f"Test {'passed' if success else 'failed'}") - return success + try: + args = (EndpointRestriction.endpoint_code == "TEST001",) + new_arg = EndpointRestriction.endpoint_method == "GET" + + updated_args = EndpointRestriction.add_new_arg_to_args(args, "endpoint_method", new_arg) + success = len(updated_args) == 2 + + # Test duplicate prevention + duplicate_arg = EndpointRestriction.endpoint_method == "GET" + updated_args = EndpointRestriction.add_new_arg_to_args(updated_args, "endpoint_method", duplicate_arg) + success = success and len(updated_args) == 2 # Should not add duplicate + + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def test_produce_query_to_add(): """Test adding query parameters to filter options.""" print("\nTesting produce_query_to_add...") with EndpointRestriction.new_session() as db_session: - sample_endpoint = create_sample_endpoint_restriction() - filter_list = { - "query": { - "endpoint_method": "GET", - "endpoint_code": "TEST001" + try: + sample_endpoint = create_sample_endpoint_restriction("TEST001") + filter_list = { + "query": { + "endpoint_method": "GET", + "endpoint_code": "TEST001" + } } - } - args = () - - updated_args = EndpointRestriction.produce_query_to_add(filter_list, args) - success = len(updated_args) == 2 - - result = EndpointRestriction.filter_all( - *updated_args, - db=db_session - ) - - # Test PostgresResponse properties - success = ( - success and - result.count == 1 and - result.total_count == 1 and - len(result.data) == 1 and - result.data[0].endpoint_code == "TEST001" and - result.is_list is True and - isinstance(result.data_as_dict, list) and - len(result.data_as_dict) == 1 and - result.data_as_dict[0].get("endpoint_code") == "TEST001" - ) - - print(f"Test {'passed' if success else 'failed'}") - return success + args = () + + updated_args = EndpointRestriction.produce_query_to_add(filter_list, args) + success = len(updated_args) == 2 + + result = EndpointRestriction.filter_all( + *updated_args, + db=db_session + ) + + # Test PostgresResponse properties + success = ( + success and + result is not None and + result.count == 1 and + result.total_count == 1 and + result.is_list is True + ) + + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False + +def test_get_dict(): + """Test the get_dict() function for single-record filters.""" + print("\nTesting get_dict...") + with EndpointRestriction.new_session() as db_session: + try: + # Set up pre_query first + EndpointRestriction.pre_query = EndpointRestriction.filter_all( + EndpointRestriction.endpoint_method == "GET", + db=db_session + ).query + + # Create a sample endpoint + endpoint_code = "TEST_DICT_001" + sample_endpoint = create_sample_endpoint_restriction(endpoint_code) + + # Get the endpoint using filter_one + result = EndpointRestriction.filter_one( + EndpointRestriction.endpoint_code == endpoint_code, + db=db_session + ) + + # Get the data and convert to dict + data = result.data + data_dict = data.get_dict() + + # Test dictionary properties + success = ( + data_dict is not None and + isinstance(data_dict, dict) and + data_dict.get("endpoint_code") == endpoint_code and + data_dict.get("endpoint_method") == "GET" and + data_dict.get("endpoint_function") == "test_function" and + data_dict.get("endpoint_name") == "Test Endpoint" and + data_dict.get("endpoint_desc") == "Test Description" and + data_dict.get("is_confirmed") is True and + data_dict.get("active") is True and + data_dict.get("deleted") is False + ) + + print(f"Test {'passed' if success else 'failed'}") + return success + except Exception as e: + print(f"Test failed with exception: {e}") + return False def run_all_tests(): """Run all tests and report results.""" print("Starting EndpointRestriction tests...") + + # Clean up any existing test data before starting + cleanup_test_data() + tests = [ test_filter_by_one, test_filter_by_one_system, @@ -318,22 +441,45 @@ def run_all_tests(): test_filter_by_all_system, test_get_not_expired_query_arg, test_add_new_arg_to_args, - test_produce_query_to_add + test_produce_query_to_add, + test_get_dict # Added new test ] - - passed = 0 - failed = 0 - + passed_list, not_passed_list = [], [] + passed, failed = 0, 0 + for test in tests: - if test(): - passed += 1 - else: + # Clean up test data before each test + cleanup_test_data() + try: + if test(): + passed += 1 + passed_list.append( + f"Test {test.__name__} passed" + ) + else: + failed += 1 + not_passed_list.append( + f"Test {test.__name__} failed" + ) + except Exception as e: + print(f"Test {test.__name__} failed with exception: {e}") failed += 1 - - print(f"\nTest Summary:") - print(f"Total tests: {len(tests)}") - print(f"Passed: {passed}") - print(f"Failed: {failed}") + not_passed_list.append( + f"Test {test.__name__} failed" + ) + + print(f"\nTest Results: {passed} passed, {failed} failed") + print('Passed Tests:') + print( + "\n".join(passed_list) + ) + print('Failed Tests:') + print( + "\n".join(not_passed_list) + ) + + return passed, failed if __name__ == "__main__": + generate_table_in_postgres() run_all_tests() diff --git a/Controllers/Postgres/mixin.py b/Controllers/Postgres/mixin.py index 2ee2ea2..5f60605 100644 --- a/Controllers/Postgres/mixin.py +++ b/Controllers/Postgres/mixin.py @@ -1,4 +1,5 @@ import arrow + from sqlalchemy import ( TIMESTAMP, func, @@ -14,10 +15,10 @@ from sqlalchemy_mixins.serialize import SerializeMixin from sqlalchemy_mixins.repr import ReprMixin from sqlalchemy_mixins.smartquery import SmartQueryMixin -from base import BaseAlchemyModel -from crud import CRUDModel -from filter import QueryModel -from database import Base +from Controllers.Postgres.base import BaseAlchemyModel +from Controllers.Postgres.crud import CRUDModel +from Controllers.Postgres.filter import QueryModel +from Controllers.Postgres.database import Base class BasicMixin( diff --git a/Controllers/Postgres/response.py b/Controllers/Postgres/response.py index dc752dc..0cfeff2 100644 --- a/Controllers/Postgres/response.py +++ b/Controllers/Postgres/response.py @@ -75,12 +75,12 @@ class PostgresResponse(Generic[T]): return self._query.count() @property - def query(self) -> str: + def core_query(self) -> str: """Get query object.""" return str(self._query) @property - def core_query(self) -> Query: + def query(self) -> Query: """Get query object.""" return self._query diff --git a/Controllers/Postgres/schema.py b/Controllers/Postgres/schema.py index e8f3a3b..a252857 100644 --- a/Controllers/Postgres/schema.py +++ b/Controllers/Postgres/schema.py @@ -1,7 +1,7 @@ from sqlalchemy import String from sqlalchemy.orm import mapped_column, Mapped -from mixin import CrudCollection +from Controllers.Postgres.mixin import CrudCollection class EndpointRestriction(CrudCollection): diff --git a/Controllers/Redis/base.py b/Controllers/Redis/base.py index bada00c..d43d443 100644 --- a/Controllers/Redis/base.py +++ b/Controllers/Redis/base.py @@ -10,10 +10,10 @@ This module provides a class for managing Redis key-value operations with suppor import arrow import json - -from connection import redis_cli from typing import Union, Dict, List, Optional, Any, TypeVar +from Controllers.Redis.connection import redis_cli + T = TypeVar('T', Dict[str, Any], List[Any]) diff --git a/Controllers/Redis/config.py b/Controllers/Redis/config.py index e69de29..e4c3462 100644 --- a/Controllers/Redis/config.py +++ b/Controllers/Redis/config.py @@ -0,0 +1,25 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Configs(BaseSettings): + """ + MongoDB configuration settings. + """ + HOST: str = "" + PASSWORD: str = "" + PORT: int = 0 + DB: int = 0 + + def as_dict(self): + return dict( + host=self.HOST, + password=self.PASSWORD, + port=int(self.PORT), + db=self.DB, + ) + + model_config = SettingsConfigDict(env_prefix="REDIS_") + + +redis_configs = Configs() # singleton instance of the REDIS configuration settings +print(redis_configs.as_dict()) diff --git a/Controllers/Redis/connection.py b/Controllers/Redis/connection.py index a756d75..ac302c6 100644 --- a/Controllers/Redis/connection.py +++ b/Controllers/Redis/connection.py @@ -1,7 +1,8 @@ import time -from typing import Dict, Any, Optional -from redis import Redis, ConnectionError, TimeoutError +from typing import Dict, Any +from redis import Redis, ConnectionError, TimeoutError, ConnectionPool +from Controllers.Redis.config import redis_configs class RedisConn: @@ -16,18 +17,16 @@ class RedisConn: def __init__( self, - config: Optional[Dict[str, Any]] = None, max_retries: int = CONNECTION_RETRIES, ): """ Initialize Redis connection with configuration. Args: - config: Redis connection configuration dictionary. If None, uses WagRedis config. max_retries: Maximum number of connection attempts. """ self.max_retries = max_retries - self.config = config or {} + self.config = redis_configs.as_dict() self._redis = None self._pool = None @@ -72,7 +71,6 @@ class RedisConn: for attempt in range(1, self.max_retries + 1): try: if self._pool is None: - from redis import ConnectionPool self._pool = ConnectionPool(**self.config) self._redis = Redis(connection_pool=self._pool) if self.check_connection(): @@ -101,7 +99,7 @@ class RedisConn: return False def set_connection( - self, host: str, password: str, port: int, db: int, **kwargs + self, **kwargs ) -> Redis: """ Recreate Redis connection with new parameters. @@ -119,10 +117,10 @@ class RedisConn: try: # Update configuration self.config = { - "host": host, - "password": password, - "port": port, - "db": db, + "host": redis_configs.HOST, + "password": redis_configs.PASSWORD, + "port": redis_configs.PORT, + "db": redis_configs.PORT, "socket_timeout": kwargs.get("socket_timeout", self.DEFAULT_TIMEOUT), "socket_connect_timeout": kwargs.get( "socket_connect_timeout", self.DEFAULT_TIMEOUT diff --git a/Controllers/Redis/database.py b/Controllers/Redis/database.py index 901c704..d4eafeb 100644 --- a/Controllers/Redis/database.py +++ b/Controllers/Redis/database.py @@ -1,9 +1,10 @@ import arrow from typing import Optional, List, Dict, Union, Iterator -from response import RedisResponse -from connection import redis_cli -from base import RedisRow + +from Controllers.Redis.response import RedisResponse +from Controllers.Redis.connection import redis_cli +from Controllers.Redis.base import RedisRow class MainConfig: diff --git a/Controllers/Redis/implementations.py b/Controllers/Redis/implementations.py index ee8016a..14bf92d 100644 --- a/Controllers/Redis/implementations.py +++ b/Controllers/Redis/implementations.py @@ -1,5 +1,5 @@ -from typing import Dict, List, Optional -from database import RedisActions +from Controllers.Redis.database import RedisActions + def example_set_json() -> None: """Example of setting JSON data in Redis with and without expiry.""" @@ -7,48 +7,48 @@ def example_set_json() -> None: data = {"name": "John", "age": 30, "city": "New York"} keys = ["user", "profile", "123"] result = RedisActions.set_json(list_keys=keys, value=data) - print("Set JSON without expiry:", result) + print("Set JSON without expiry:", result.as_dict()) # Example 2: Set JSON with expiry expiry = {"hours": 1, "minutes": 30} result = RedisActions.set_json(list_keys=keys, value=data, expires=expiry) - print("Set JSON with expiry:", result) + print("Set JSON with expiry:", result.as_dict()) def example_get_json() -> None: """Example of retrieving JSON data from Redis.""" # Example 1: Get all matching keys keys = ["user", "profile", "*"] result = RedisActions.get_json(list_keys=keys) - print("Get all matching JSON:", result) + print("Get all matching JSON:", result.as_dict()) # Example 2: Get with limit result = RedisActions.get_json(list_keys=keys, limit=5) - print("Get JSON with limit:", result) + print("Get JSON with limit:", result.as_dict()) def example_get_json_iterator() -> None: """Example of using the JSON iterator for large datasets.""" keys = ["user", "profile", "*"] for row in RedisActions.get_json_iterator(list_keys=keys): - print("Iterating over JSON row:", row) + print("Iterating over JSON row:", row.as_dict if isinstance(row.as_dict, dict) else row.as_dict()) -def example_delete_key() -> None: - """Example of deleting a specific key.""" - key = "user:profile:123" - result = RedisActions.delete_key(key) - print("Delete specific key:", result) - -def example_delete() -> None: - """Example of deleting multiple keys matching a pattern.""" - keys = ["user", "profile", "*"] - result = RedisActions.delete(list_keys=keys) - print("Delete multiple keys:", result) +# def example_delete_key() -> None: +# """Example of deleting a specific key.""" +# key = "user:profile:123" +# result = RedisActions.delete_key(key) +# print("Delete specific key:", result) +# +# def example_delete() -> None: +# """Example of deleting multiple keys matching a pattern.""" +# keys = ["user", "profile", "*"] +# result = RedisActions.delete(list_keys=keys) +# print("Delete multiple keys:", result) def example_refresh_ttl() -> None: """Example of refreshing TTL for a key.""" key = "user:profile:123" new_expiry = {"hours": 2, "minutes": 0} result = RedisActions.refresh_ttl(key=key, expires=new_expiry) - print("Refresh TTL:", result) + print("Refresh TTL:", result.as_dict()) def example_key_exists() -> None: """Example of checking if a key exists.""" @@ -58,9 +58,10 @@ def example_key_exists() -> None: def example_resolve_expires_at() -> None: """Example of resolving expiry time for a key.""" - from base import RedisRow + from Controllers.Redis.base import RedisRow redis_row = RedisRow() - redis_row.redis_key = "user:profile:123" + redis_row.set_key("user:profile:123") + print(redis_row.keys) expires_at = RedisActions.resolve_expires_at(redis_row) print("Resolve expires at:", expires_at) @@ -77,11 +78,11 @@ def run_all_examples() -> None: print("\n3. Using JSON iterator:") example_get_json_iterator() - print("\n4. Deleting specific key:") - example_delete_key() - - print("\n5. Deleting multiple keys:") - example_delete() + # print("\n4. Deleting specific key:") + # example_delete_key() + # + # print("\n5. Deleting multiple keys:") + # example_delete() print("\n6. Refreshing TTL:") example_refresh_ttl() diff --git a/Controllers/Redis/response.py b/Controllers/Redis/response.py index 6a3f021..91d3a57 100644 --- a/Controllers/Redis/response.py +++ b/Controllers/Redis/response.py @@ -1,5 +1,5 @@ -from typing import Union, Dict, List, Optional, Any -from base import RedisRow +from typing import Union, Dict, Optional, Any +from Controllers.Redis.base import RedisRow class RedisResponse: @@ -11,11 +11,11 @@ class RedisResponse: """ def __init__( - self, - status: bool, - message: str, - data: Any = None, - error: Optional[str] = None, + self, + status: bool, + message: str, + data: Any = None, + error: Optional[str] = None, ): """ Initialize a Redis response. diff --git a/api_env.env b/api_env.env index d8afcda..f6c8b34 100644 --- a/api_env.env +++ b/api_env.env @@ -1,18 +1,22 @@ MONGO_ENGINE=mongodb -MONGO_DB=mongodb +MONGO_DB=mongo_database MONGO_HOST=mongo_service MONGO_PORT=27017 MONGO_USER=mongo_user MONGO_PASSWORD=mongo_password -POSTGRES_DB=POSTGRES_DB -POSTGRES_USER=POSTGRES_USER -POSTGRES_PASSWORD=POSTGRES_PASSWORD -POSTGRES_HOST=POSTGRES_HOST -POSTGRES_PORT=1155 -POSTGRES_ENGINE=POSTGRES_ENGINE +POSTGRES_DB=wag_database +POSTGRES_USER=berkay_wag_user +POSTGRES_PASSWORD=berkay_wag_user_password +POSTGRES_HOST=postgres-service +POSTGRES_PORT=5432 +POSTGRES_ENGINE=postgresql+psycopg2 POSTGRES_POOL_PRE_PING=True POSTGRES_POOL_SIZE=20 POSTGRES_MAX_OVERFLOW=10 POSTGRES_POOL_RECYCLE=600 POSTGRES_POOL_TIMEOUT=30 -POSTGRES_ECHO=True \ No newline at end of file +POSTGRES_ECHO=True +REDIS_HOST=redis_service +REDIS_PASSWORD=commercial_redis_password +REDIS_PORT=6379 +REDIS_DB=0 \ No newline at end of file diff --git a/debian-docker-compose.yml b/debian-docker-compose.yml new file mode 100644 index 0000000..89143de --- /dev/null +++ b/debian-docker-compose.yml @@ -0,0 +1,9 @@ +services: + + test_server: + container_name: test_server + build: + context: . + dockerfile: ApiServices/Dockerfile + env_file: + - api_env.env diff --git a/docker-compose.yml b/docker-compose.yml index 89143de..1fb7175 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,53 @@ services: + mongo_service: + container_name: mongo_service + image: "bitnami/mongodb:latest" + networks: + - wag-services + environment: + - MONGODB_DISABLE_ENFORCE_AUTH=true + - MONGODB_ROOT_PASSWORD=root + - MONGODB_DATABASE=mongo_database + - MONGODB_USERNAME=mongo_user + - MONGODB_PASSWORD=mongo_password + - MONGO_INITDB_ROOT_USERNAME=mongo_user + - MONGO_INITDB_ROOT_PASSWORD=mongo_password + - MONGO_INITDB_DATABASE=mongo_database + ports: + - "11777:27017" + volumes: + - mongodb-data:/bitnami/mongodb + + postgres-service: + container_name: postgres-service + image: "bitnami/postgresql:latest" + networks: + - wag-services + restart: on-failure + depends_on: + - mongo_service + environment: + - POSTGRES_DB=wag_database + - POSTGRES_USER=berkay_wag_user + - POSTGRES_PASSWORD=berkay_wag_user_password + ports: + - "5444:5432" + volumes: + - postgres-data:/bitnami/postgresql + + redis_service: + container_name: redis_service + image: "bitnami/redis:latest" + networks: + - wag-services + restart: on-failure + environment: + - REDIS_HOST=redis_service + - REDIS_PASSWORD=commercial_redis_password + - REDIS_PORT=6379 + - REDIS_DB=0 + ports: + - "11222:6379" test_server: container_name: test_server @@ -7,3 +56,12 @@ services: dockerfile: ApiServices/Dockerfile env_file: - api_env.env + networks: + - wag-services + +networks: + wag-services: + +volumes: + postgres-data: + mongodb-data: