diff --git a/.gitignore b/.gitignore index f8ab384..dba3cf9 100644 --- a/.gitignore +++ b/.gitignore @@ -164,4 +164,4 @@ yarn-error.log* next-env.d.ts # Project specific -ServicesBank/Zemberek/ \ No newline at end of file +trash/Zemberek/ \ No newline at end of file diff --git a/ServicesApi/Builds/Auth/events/auth/events.py b/ServicesApi/Builds/Auth/events/auth/events.py index 4e2404c..9a69314 100644 --- a/ServicesApi/Builds/Auth/events/auth/events.py +++ b/ServicesApi/Builds/Auth/events/auth/events.py @@ -386,7 +386,7 @@ class LoginHandler: ).join(BuildParts, BuildParts.id == BuildLivingSpace.build_parts_id ).join(Build, Build.id == BuildParts.build_id ).join(People, People.id == BuildLivingSpace.person_id - ).filter(BuildLivingSpace.uu_id == data.uuid) + ).filter(BuildLivingSpace.uu_id == data.uuid) selected_build_living_space_first = selected_build_living_space_query.first() if not selected_build_living_space_first: diff --git a/ServicesApi/Builds/Initial/app.py b/ServicesApi/Builds/Initial/app.py index 554067b..8819c74 100644 --- a/ServicesApi/Builds/Initial/app.py +++ b/ServicesApi/Builds/Initial/app.py @@ -17,7 +17,7 @@ if __name__ == "__main__": with get_db() as db_session: if set_alembic: generate_alembic(session=db_session) - + exit() try: create_one_address(db_session=db_session) except Exception as e: diff --git a/ServicesApi/Extensions/Middlewares/token_provider.py b/ServicesApi/Extensions/Middlewares/token_provider.py index 93e32ab..bf80569 100644 --- a/ServicesApi/Extensions/Middlewares/token_provider.py +++ b/ServicesApi/Extensions/Middlewares/token_provider.py @@ -26,7 +26,6 @@ class TokenProvider: return OccupantTokenObject(**redis_object) raise ValueError("Invalid user type") - @classmethod def get_login_token_from_redis( cls, token: Optional[str] = None, user_uu_id: Optional[str] = None diff --git a/ServicesApi/Schemas/__init__.py b/ServicesApi/Schemas/__init__.py index 8eacc2f..c64d3df 100644 --- a/ServicesApi/Schemas/__init__.py +++ b/ServicesApi/Schemas/__init__.py @@ -7,6 +7,8 @@ from Schemas.account.account import ( AccountRecordExchanges, AccountRecords, AccountDelayInterest, + AccountRecordsPredict, + AccountRecordsModelTrain, ) from Schemas.account.iban import ( BuildIbans, @@ -126,6 +128,8 @@ __all__ = [ "AccountRecordExchanges", "AccountRecords", "AccountDelayInterest", + "AccountRecordsPredict", + "AccountRecordsModelTrain", "BuildIbans", "BuildIbanDescription", "RelationshipEmployee2PostCode", diff --git a/ServicesApi/Schemas/account/account.py b/ServicesApi/Schemas/account/account.py index 1428c78..f284cd4 100644 --- a/ServicesApi/Schemas/account/account.py +++ b/ServicesApi/Schemas/account/account.py @@ -1,6 +1,7 @@ from Schemas.base_imports import ( CrudCollection, String, + Text, Integer, Boolean, ForeignKey, @@ -384,9 +385,9 @@ class AccountRecordsPredict(CrudCollection): account_records_id: Mapped[int] = mapped_column(ForeignKey("account_records.id"), nullable=False) account_records_uu_id: Mapped[str] = mapped_column(String(100), nullable=False) - prediction_model: Mapped[str] = mapped_column(String(10), nullable=False) - prediction_result: Mapped[int] = mapped_column(Integer, nullable=False) - prediction_field: Mapped[str] = mapped_column(String(10), nullable=False, server_default="") + prediction_model: Mapped[str] = mapped_column(String, nullable=False) + prediction_result: Mapped[str] = mapped_column(Text, nullable=False) + prediction_field: Mapped[str] = mapped_column(String, nullable=False, server_default="") treshold: Mapped[float] = mapped_column(Numeric(18, 6), nullable=True) is_first_prediction: Mapped[bool] = mapped_column(Boolean, server_default="0") is_approved: Mapped[bool] = mapped_column(Boolean, server_default="0") diff --git a/ServicesApi/Schemas/identity/identity.py b/ServicesApi/Schemas/identity/identity.py index 0e1d868..220aee9 100644 --- a/ServicesApi/Schemas/identity/identity.py +++ b/ServicesApi/Schemas/identity/identity.py @@ -211,49 +211,22 @@ class People(CrudCollection): "tax_no", ] - firstname: Mapped[str] = mapped_column( - String, nullable=False, comment="First name of the person" - ) - surname: Mapped[str] = mapped_column( - String(24), nullable=False, comment="Surname of the person" - ) - middle_name: Mapped[str] = mapped_column( - String, server_default="", comment="Middle name of the person" - ) - sex_code: Mapped[str] = mapped_column( - String(1), nullable=False, comment="Sex code of the person (e.g., M/F)" - ) - person_ref: Mapped[str] = mapped_column( - String, server_default="", comment="Reference ID for the person" - ) - person_tag: Mapped[str] = mapped_column( - String, server_default="", comment="Unique tag for the person" - ) + firstname: Mapped[str] = mapped_column(String, nullable=False, comment="First name of the person") + surname: Mapped[str] = mapped_column(String(24), nullable=False, comment="Surname of the person") + middle_name: Mapped[str] = mapped_column(String, server_default="", comment="Middle name of the person") + birthname: Mapped[str] = mapped_column(String, nullable=True, server_default="", comment="Birth name of the person") + sex_code: Mapped[str] = mapped_column(String(1), nullable=False, comment="Sex code of the person (e.g., M/F)") + person_ref: Mapped[str] = mapped_column(String, server_default="", comment="Reference ID for the person") + person_tag: Mapped[str] = mapped_column(String, server_default="", comment="Unique tag for the person") # ENCRYPT DATA - father_name: Mapped[str] = mapped_column( - String, server_default="", comment="Father's name of the person" - ) - mother_name: Mapped[str] = mapped_column( - String, server_default="", comment="Mother's name of the person" - ) - country_code: Mapped[str] = mapped_column( - String(4), server_default="TR", comment="Country code of the person" - ) - national_identity_id: Mapped[str] = mapped_column( - String, server_default="", comment="National identity ID of the person" - ) - birth_place: Mapped[str] = mapped_column( - String, server_default="", comment="Birth place of the person" - ) - birth_date: Mapped[TIMESTAMP] = mapped_column( - TIMESTAMP(timezone=True), - server_default="1900-01-01", - comment="Birth date of the person", - ) - tax_no: Mapped[str] = mapped_column( - String, server_default="", comment="Tax number of the person" - ) + father_name: Mapped[str] = mapped_column(String, server_default="", comment="Father's name of the person") + mother_name: Mapped[str] = mapped_column(String, server_default="", comment="Mother's name of the person") + country_code: Mapped[str] = mapped_column(String(4), server_default="TR", comment="Country code of the person") + national_identity_id: Mapped[str] = mapped_column(String, server_default="", comment="National identity ID of the person") + birth_place: Mapped[str] = mapped_column(String, server_default="", comment="Birth place of the person") + birth_date: Mapped[TIMESTAMP] = mapped_column(TIMESTAMP(timezone=True), server_default="1900-01-01", comment="Birth date of the person") + tax_no: Mapped[str] = mapped_column(String, server_default="", comment="Tax number of the person") # Receive at Create person # language = mapped_column( # String, comment="Language code of the person" @@ -262,10 +235,7 @@ class People(CrudCollection): # String, comment="Currency code of the person" # ) - # ENCRYPT DATA - user = relationship( - "Users", back_populates="person", foreign_keys="Users.person_id" - ) + user = relationship("Users", back_populates="person", foreign_keys="Users.person_id") __table_args__ = ( Index( diff --git a/ServicesBank/Depends/config.py b/ServicesBank/Depends/config.py new file mode 100644 index 0000000..2a9b87e --- /dev/null +++ b/ServicesBank/Depends/config.py @@ -0,0 +1,31 @@ +import os + + +class Config: + + MAILBOX: str = os.getenv("MAILBOX", "bilgilendirme@ileti.isbank.com.tr") + MAIN_MAIL: str = os.getenv("MAIN_MAIL", "karatay.berkay@gmail.com") + INFO_MAIL: str = os.getenv("INFO_MAIL", "mehmet.karatay@hotmail.com") + EMAIL_HOST: str = os.getenv("EMAIL_HOST", "10.10.2.34") + EMAIL_SENDER_USERNAME: str = os.getenv("EMAIL_SENDER_USERNAME", "karatay@mehmetkaratay.com.tr") + EMAIL_USERNAME: str = os.getenv("EMAIL_USERNAME", "isbank@mehmetkaratay.com.tr") + EMAIL_PASSWORD: str = os.getenv("EMAIL_PASSWORD", "system") + AUTHORIZE_IBAN: str = os.getenv("AUTHORIZE_IBAN", "4245-0093333") + SERVICE_TIMING: int = int(os.getenv("SERVICE_TIMING", 900)) + EMAIL_PORT: int = int(os.getenv("EMAIL_PORT", 993)) + EMAIL_SEND_PORT: int = int(os.getenv("EMAIL_SEND_PORT", 587)) + EMAIL_SLEEP: int = int(os.getenv("EMAIL_SLEEP", 60)) + EMAIL_SEND: bool = bool(os.getenv("EMAIL_SEND", False)) + + +class EmailConfig: + + EMAIL_HOST: str = os.getenv("EMAIL_HOST", "10.10.2.34") + EMAIL_USERNAME: str = Config.EMAIL_SENDER_USERNAME + EMAIL_PASSWORD: str = Config.EMAIL_PASSWORD + EMAIL_PORT: int = Config.EMAIL_SEND_PORT + EMAIL_SEND: bool = Config.EMAIL_SEND + + @classmethod + def as_dict(cls): + return dict(host=EmailConfig.EMAIL_HOST, port=EmailConfig.EMAIL_PORT, username=EmailConfig.EMAIL_USERNAME, password=EmailConfig.EMAIL_PASSWORD) diff --git a/ServicesBank/Depends/template_accounts.html b/ServicesBank/Depends/template_accounts.html new file mode 100644 index 0000000..67c03d1 --- /dev/null +++ b/ServicesBank/Depends/template_accounts.html @@ -0,0 +1,54 @@ + + + + + + Gelen Banka Kayıtları + + + +

Günaydın, Admin

+
+

Banka Kayıtları : {{today}}

+

Son Bakiye : {{bank_balance}}

+

{{"Status : İkinci Bakiye Hatalı" if balance_error else "Status :OK"}}

+ + + + {% for header in headers %} + + {% endfor %} + + + + {% for row in rows %} + + {% for cell in row %} + + {% endfor %} + + {% endfor %} + +
{{ header }}
{{ cell }}
+

Teşekkür ederiz,
Evyos Yönetim
Saygılarımızla

+ + \ No newline at end of file diff --git a/ServicesBank/Email/Dockerfile b/ServicesBank/Email/Dockerfile new file mode 100644 index 0000000..7615823 --- /dev/null +++ b/ServicesBank/Email/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR / + +RUN apt-get update && apt-get install -y --no-install-recommends gcc && rm -rf /var/lib/apt/lists/* && pip install --no-cache-dir poetry + +COPY /ServicesBank/Email/pyproject.toml ./pyproject.toml + +RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main && pip cache purge && rm -rf ~/.cache/pypoetry + +COPY /ServicesBank/Email / +COPY /ServicesApi/Controllers /ServicesApi/Controllers +COPY /ServicesBank/Depends/config.py /ServicesBank/Depends/config.py + +ENV PYTHONPATH=/ PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 + +CMD ["poetry", "run", "python", "app.py"] diff --git a/ServicesBank/Email/README.md b/ServicesBank/Email/README.md new file mode 100644 index 0000000..4647acf --- /dev/null +++ b/ServicesBank/Email/README.md @@ -0,0 +1,84 @@ +# Email Service + +## Overview +The Email Service is the first component in a Redis pub/sub processing chain for bank-related email automation. It monitors a specified mailbox for emails with attachments, filters them based on IBAN criteria, and publishes the data to a Redis channel for further processing. + +## Features + +### Email Processing +- Connects to a configured mailbox using IMAP +- Implements smart date-based filtering: + - Checks emails from the past 14 days on the first run of each day + - Checks emails from the past 7 days on subsequent runs within the same day +- Extracts attachments from emails +- Filters attachments based on IBAN criteria +- Uses a context manager to ensure emails are properly handled even during errors + +### Redis Integration +- Publishes messages to a Redis pub/sub channel ("CollectedData") +- Each message contains: + - Unique UUID + - Timestamp + - Initial stage marker ("red") + - Attachment payload and metadata +- Connects to an external Redis server + +### Error Handling +- Robust error management with context managers +- Automatic marking of emails as unread if processing fails +- Comprehensive logging + +## Configuration + +### Environment Variables +``` +EMAIL_HOST=10.10.2.34 +EMAIL_USERNAME=isbank@mehmetkaratay.com.tr +EMAIL_PASSWORD=system +EMAIL_SLEEP=60 +AUTHORIZE_IBAN=4245-0093333 +REDIS_HOST=10.10.2.15 +REDIS_PORT=6379 +REDIS_PASSWORD=your_strong_password_here +``` + +## Deployment + +### Docker +The service is containerized using Docker and can be deployed using the provided Dockerfile and docker-compose configuration. + +```bash +# Build and start the service +docker compose -f bank-services-docker-compose.yml up -d --build + +# View logs +docker compose -f bank-services-docker-compose.yml logs -f email_service + +# Stop the service +docker compose -f bank-services-docker-compose.yml down +``` + +### Service Management +The `check_bank_services.sh` script provides a simple way to restart the service: + +```bash +./check_bank_services.sh +``` + +## Architecture + +### Redis Pub/Sub Chain +This service is the first in a multi-stage processing chain: +1. **Email Service** (this service): Reads emails, extracts attachments, publishes to Redis with stage="red" +2. **Processor Service**: Subscribes to stage="red" messages, processes data, republishes with stage="processed" +3. **Writer Service**: Subscribes to stage="processed" messages, writes data to final destination, marks as stage="completed" + +## Development + +### Dependencies +- Python 3.12 +- Redbox (email library) +- Redis + +### State Management +The service maintains a state file at `/tmp/email_service_last_run.json` to track when it last ran, enabling the smart date-based filtering feature. diff --git a/ServicesBank/Email/app.py b/ServicesBank/Email/app.py new file mode 100644 index 0000000..0c65776 --- /dev/null +++ b/ServicesBank/Email/app.py @@ -0,0 +1,169 @@ +import time +import arrow +import os +import json +import base64 + +from uuid import uuid4 +from datetime import datetime, timedelta +from typing import TypeVar +from redbox import EmailBox +from redbox.query import FROM, UNSEEN, OR, SINCE + +from ServicesApi.Controllers.Redis.Broadcast.actions import redis_pubsub +from ServicesBank.Depends.config import Config + + +authorized_iban = Config.AUTHORIZE_IBAN +authorized_iban_cleaned = authorized_iban.replace("-", "") +REDIS_CHANNEL = "reader" +delimiter = "|" + +# banks_mails = mailbox.search(from_=filter_mail, unseen=True) bununla denemeyin +# banks_mails = mailbox.search(FROM(filter_mail) & UNSEEN) + + +T = TypeVar("T") + + +class EmailProcessingContext: + """Context manager for email processing that marks emails as unread if an error occurs.""" + + def __init__(self, email_message, mark_as_read: bool = True): + self.email_message = email_message + self.mark_as_read = mark_as_read + self.success = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None or not self.success: + try: + if hasattr(self.email_message, "mark_as_unread"): + self.email_message.mark_as_unread() + print(f"[EMAIL_SERVICE] Marked email as UNREAD due to processing error: {exc_val if exc_val else 'Unknown error'}") + except Exception as e: + print(f"[EMAIL_SERVICE] Failed to mark email as unread: {str(e)}") + elif self.mark_as_read: + try: + if hasattr(self.email_message, "mark_as_read"): + self.email_message.mark_as_read() + except Exception as e: + print(f"[EMAIL_SERVICE] Failed to mark email as read: {str(e)}") + return False + + +def publish_payload_to_redis(payload, filename: str, mail_info: dict) -> bool: + if isinstance(payload, bytes): + encoded_payload = base64.b64encode(payload).decode("utf-8") + is_base64 = True + else: + encoded_payload = payload + is_base64 = False + + message = { + "filename": filename, "payload": encoded_payload, "is_base64": is_base64, "stage": "red", + "created_at": str(arrow.now()), "uuid": str(uuid4()), **mail_info, + } + + result = redis_pubsub.publisher.publish(REDIS_CHANNEL, message) + if result.status: + print(f"[EMAIL_SERVICE] Published message with filename: {filename} to channel: {REDIS_CHANNEL}") + return True + else: + print(f"[EMAIL_SERVICE] Publish error: {result.error}") + return False + + +def read_email_and_publish_to_redis(email_message, mail_info: dict) -> bool: + if email_message.is_multipart(): # Check if email has multipart content + for part in email_message.walk(): # Each part can be an attachment + content_disposition = part.get("Content-Disposition") + if content_disposition and "attachment" in content_disposition: + if filename := part.get_filename(): + is_iban_in_filename = authorized_iban_cleaned in str(filename) + if is_iban_in_filename: + if payload := part.get_payload(decode=True): + return publish_payload_to_redis(payload=payload, filename=filename, mail_info=mail_info) + else: + content_disposition = email_message.get("Content-Disposition") + if content_disposition and "attachment" in content_disposition: + if filename := email_message.get_filename(): + is_iban_in_filename = authorized_iban_cleaned in str(filename) + if is_iban_in_filename: + payload = email_message.get_payload(decode=True) + return publish_payload_to_redis(payload=payload, filename=filename, mail_info=mail_info) + return False + + +def app(): + + host = Config.EMAIL_HOST + port = Config.EMAIL_PORT + username = Config.EMAIL_USERNAME + password = Config.EMAIL_PASSWORD + + box = EmailBox(host=host, port=port, username=username, password=password) + if not box: + return Exception("Mailbox not found") + + box.connect() + mail_folders = box.mailfolders + filter_mail = OR(FROM(Config.MAILBOX), FROM(Config.MAIN_MAIL)) + filter_print = f"{Config.MAILBOX} & {Config.MAIN_MAIL}" + last_run_file = "/tmp/email_service_last_run.json" + current_date = datetime.now().strftime("%Y-%m-%d") + days_to_check, full_check = 7, 90 + + try: + if os.path.exists(last_run_file): + with open(last_run_file, "r") as f: + last_run_data = json.load(f) + last_run_date = last_run_data.get("last_run_date") + if last_run_date != current_date: + days_to_check = full_check + print(f"[EMAIL_SERVICE] First run of the day. Checking emails from the past {days_to_check} days") + else: + print(f"[EMAIL_SERVICE] Subsequent run today. Checking emails from the past {days_to_check} days") + else: + days_to_check = full_check + print(f"[EMAIL_SERVICE] First run detected. Checking emails from the past {days_to_check} days") + except Exception as e: + print(f"[EMAIL_SERVICE] Error reading last run file: {str(e)}. Using default of {days_to_check} days") + + try: + with open(last_run_file, "w") as f: + json.dump({"last_run_date": current_date}, f) + except Exception as e: + print(f"[EMAIL_SERVICE] Error writing last run file: {str(e)}") + + check_since_date = (datetime.now() - timedelta(days=days_to_check)).strftime("%d-%b-%Y") + for folder in mail_folders: + if folder.name == "INBOX": + banks_mails = folder.search(filter_mail & SINCE(check_since_date)) + print(f"[EMAIL_SERVICE] Reading mailbox [{username}] with mail sender [{filter_print}] since {check_since_date} with count: {len(banks_mails)}") + for banks_mail in banks_mails or []: + if email_message := banks_mail.email: + with EmailProcessingContext(banks_mail) as ctx: + try: + headers = {k.lower(): v for k, v in banks_mail.headers.items()} + mail_info = {"from": headers["from"], "to": headers["to"], "subject": headers["subject"], "date": str(headers["date"])} + success = read_email_and_publish_to_redis(email_message=email_message, mail_info=mail_info) + ctx.success = success + if success: + print(f"[EMAIL_SERVICE] Successfully processed email with subject: {mail_info['subject']}") + else: + print(f"[EMAIL_SERVICE] No matching attachments found in email with subject: {mail_info['subject']}") + except Exception as e: + print(f"[EMAIL_SERVICE] Error processing email: {str(e)}") + + +if __name__ == "__main__": + print("=== Starting Email Service with Redis Pub/Sub ===") + print(f"Publishing to channel: {REDIS_CHANNEL}") + time.sleep(20) + while True: + print("\n[EMAIL_SERVICE] Checking for new emails...") + app() + time.sleep(Config.EMAIL_SLEEP) diff --git a/ServicesBank/Email/pyproject.toml b/ServicesBank/Email/pyproject.toml new file mode 100644 index 0000000..e6943ae --- /dev/null +++ b/ServicesBank/Email/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "emailservice" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "arrow>=1.3.0", + "redbox>=0.2.1", + "redis>=5.2.1", + "pydantic-settings>=2.8.1", +] diff --git a/ServicesBank/Finder/BuildFromIban/.dockerignore b/ServicesBank/Finder/BuildExtractor/.dockerignore similarity index 100% rename from ServicesBank/Finder/BuildFromIban/.dockerignore rename to ServicesBank/Finder/BuildExtractor/.dockerignore diff --git a/ServicesBank/Finder/BuildExtractor/Dockerfile b/ServicesBank/Finder/BuildExtractor/Dockerfile new file mode 100644 index 0000000..f061a84 --- /dev/null +++ b/ServicesBank/Finder/BuildExtractor/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.12-slim + +WORKDIR / + +# Set Python path to include app directory +ENV PYTHONPATH=/ PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 + +# Install system dependencies and Poetry +RUN apt-get update && apt-get install -y --no-install-recommends gcc && rm -rf /var/lib/apt/lists/* && pip install --no-cache-dir poetry + +# Copy Poetry configuration +COPY /pyproject.toml ./pyproject.toml + +# Configure Poetry and install dependencies with optimizations +RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main && pip cache purge && rm -rf ~/.cache/pypoetry + +# Install cron for scheduling tasks +RUN apt-get update && apt-get install -y cron + +# Copy application code +COPY /ServicesBank/Finder/BuildExtractor / +COPY /ServicesApi/Schemas /Schemas +COPY /ServicesApi/Controllers /Controllers + +# Create log file to grab cron logs +RUN touch /var/log/cron.log + +# Make entrypoint script executable +RUN chmod +x /entrypoint.sh +RUN chmod +x /run_app.sh + +# Use entrypoint script to update run_app.sh with environment variables and start cron +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ServicesBank/Finder/BuildExtractor/README.md b/ServicesBank/Finder/BuildExtractor/README.md new file mode 100644 index 0000000..aa05ba7 --- /dev/null +++ b/ServicesBank/Finder/BuildExtractor/README.md @@ -0,0 +1,3 @@ +# Docs of Build Extractor + +Finds build_id, decision_book_id, living_space_id from AccountRecords diff --git a/ServicesBank/Finder/BuildLivingSpace/entrypoint.sh b/ServicesBank/Finder/BuildExtractor/entrypoint.sh similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/entrypoint.sh rename to ServicesBank/Finder/BuildExtractor/entrypoint.sh diff --git a/ServicesBank/Finder/BuildFromIban/run_app.sh b/ServicesBank/Finder/BuildExtractor/run_app.sh similarity index 100% rename from ServicesBank/Finder/BuildFromIban/run_app.sh rename to ServicesBank/Finder/BuildExtractor/run_app.sh diff --git a/ServicesBank/Finder/BuildExtractor/runner.py b/ServicesBank/Finder/BuildExtractor/runner.py new file mode 100644 index 0000000..d01ba03 --- /dev/null +++ b/ServicesBank/Finder/BuildExtractor/runner.py @@ -0,0 +1,66 @@ +import arrow + +from Schemas import AccountRecords, BuildIbans, BuildDecisionBook +from Controllers.Postgres.engine import get_session_factory +from sqlalchemy import cast, Date + + +def account_records_find_decision_book(session): + + AccountRecords.set_session(session) + BuildIbans.set_session(session) + BuildDecisionBook.set_session(session) + + created_ibans, iban_build_dict = [], {} + filter_account_records = AccountRecords.build_id != None, AccountRecords.build_decision_book_id == None + account_records_list: list[AccountRecords] = AccountRecords.query.filter(*filter_account_records).order_by(AccountRecords.bank_date.desc()).all() + for account_record in account_records_list: + if found_iban := BuildIbans.query.filter(BuildIbans.iban == account_record.iban).first(): + if found_decision_book := BuildDecisionBook.query.filter( + BuildDecisionBook.build_id == found_iban.build_id, + cast(BuildDecisionBook.expiry_starts, Date) <= cast(account_record.bank_date, Date), + cast(BuildDecisionBook.expiry_ends, Date) >= cast(account_record.bank_date, Date), + ).first(): + account_record.build_decision_book_id = found_decision_book.id + account_record.build_decision_book_uu_id = str(found_decision_book.uu_id) + account_record.save() + + +def account_find_build_from_iban(session): + + AccountRecords.set_session(session) + BuildIbans.set_session(session) + + account_records_ibans = AccountRecords.query.filter(AccountRecords.build_id == None, AccountRecords.approved_record == False).distinct(AccountRecords.iban).all() + for account_records_iban in account_records_ibans: + found_iban: BuildIbans = BuildIbans.query.filter(BuildIbans.iban == account_records_iban.iban).first() + if not found_iban: + create_build_ibans = BuildIbans.create(iban=account_records_iban.iban, start_date=str(arrow.now().shift(days=-1))) + create_build_ibans.save() + else: + update_dict = {"build_id": found_iban.build_id, "build_uu_id": str(found_iban.build_uu_id)} + session.query(AccountRecords).filter(AccountRecords.iban == account_records_iban.iban).update(update_dict, synchronize_session=False) + session.commit() + + +if __name__ == "__main__": + + print("Build Extractor Service is running...") + session_factory = get_session_factory() + session = session_factory() + + try: + account_find_build_from_iban(session=session) + except Exception as e: + print(f"Error occured on find build : {e}") + session.rollback() + + try: + account_records_find_decision_book(session=session) + except Exception as e: + print(f"Error occured on find decision book : {e}") + session.rollback() + + session.close() + session_factory.remove() + print("Build Extractor Service is finished...") diff --git a/ServicesBank/Finder/BuildLivingSpace/.dockerignore b/ServicesBank/Parser/.dockerignore similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/.dockerignore rename to ServicesBank/Parser/.dockerignore diff --git a/ServicesBank/Parser/Dockerfile b/ServicesBank/Parser/Dockerfile new file mode 100644 index 0000000..e915be7 --- /dev/null +++ b/ServicesBank/Parser/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12-slim + +WORKDIR / + +ENV PYTHONPATH=/ PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 + +RUN apt-get update && apt-get install -y --no-install-recommends gcc && rm -rf /var/lib/apt/lists/* && pip install --no-cache-dir poetry + +COPY pyproject.toml ./pyproject.toml + +RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main && pip cache purge && rm -rf ~/.cache/pypoetry + +RUN apt-get update && apt-get install -y cron + +COPY /ServicesBank/Parser / +COPY /ServicesApi/Schemas /Schemas +COPY /ServicesApi/Controllers /Controllers + +RUN touch /var/log/cron.log + +RUN chmod +x /entrypoint.sh +RUN chmod +x /run_app.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ServicesBank/Parser/README.md b/ServicesBank/Parser/README.md new file mode 100644 index 0000000..aa05ba7 --- /dev/null +++ b/ServicesBank/Parser/README.md @@ -0,0 +1,3 @@ +# Docs of Build Extractor + +Finds build_id, decision_book_id, living_space_id from AccountRecords diff --git a/ServicesBank/Finder/DecisionBook/entrypoint.sh b/ServicesBank/Parser/entrypoint.sh similarity index 100% rename from ServicesBank/Finder/DecisionBook/entrypoint.sh rename to ServicesBank/Parser/entrypoint.sh diff --git a/ServicesBank/Finder/DecisionBook/run_app.sh b/ServicesBank/Parser/run_app.sh similarity index 100% rename from ServicesBank/Finder/DecisionBook/run_app.sh rename to ServicesBank/Parser/run_app.sh diff --git a/ServicesBank/Parser/runner.py b/ServicesBank/Parser/runner.py new file mode 100644 index 0000000..2b9414c --- /dev/null +++ b/ServicesBank/Parser/runner.py @@ -0,0 +1,636 @@ +import re +import arrow + +from json import loads, dumps +from unidecode import unidecode +from difflib import SequenceMatcher +from itertools import permutations +from time import perf_counter +from sqlalchemy import text as sqlalchemy_text + +from Controllers.Postgres.engine import get_session_factory +from Schemas.account.account import AccountRecordsPredict, AccountRecords + + +def clean_text(text): + text = str(text) + text = re.sub(r'\d{8,}', ' ', text) + # text = re.sub(r'\b[A-Za-z0-9]*?[0-9]+[A-Za-z0-9]*?[A-Za-z]+[A-Za-z0-9]*\b|\b[A-Za-z0-9]*?[A-Za-z]+[A-Za-z0-9]*?[0-9]+[A-Za-z0-9]*\b', ' ', text) + text = text.replace("/", " ") + text = text.replace("_", " ") + text_remove_underscore = text.replace("-", " ").replace("+", " ") + text_remove_asterisk = text_remove_underscore.replace("*", " ") + text_remove_comma = text_remove_asterisk.replace(",", " ") + text_remove_dots = text_remove_comma.replace(".", " ") + text_remove_dots = re.sub(r'\s+', ' ', text_remove_dots) + text_remove_dots = text_remove_dots.strip() + return text_remove_dots + + +def normalize_text(text): + text = text.replace('İ', 'i') + text = text.replace('I', 'ı') + text = text.replace('Ş', 'ş') + text = text.replace('Ğ', 'ğ') + text = text.replace('Ü', 'ü') + text = text.replace('Ö', 'ö') + text = text.replace('Ç', 'ç') + return unidecode(text).lower() + + +def get_person_initials(person): + parts = [person.get("firstname", ""), person.get("middle_name", ""), person.get("surname", ""), person.get("birthname", "")] + return [unidecode(p.strip())[0].upper() for p in parts if p] + + +def get_text_initials(matched_text): + return [unidecode(word.strip())[0].upper() for word in matched_text.split() if word.strip()] + + +def generate_dictonary_of_patterns(people): + + """ + completly remove middle_name instead do regex firstName + SomeWord + surname + """ + patterns_dict = {} + + for person in people: + person_id = person.get('id') + firstname = person.get('firstname', '').strip() if person.get('firstname') else "" + middle_name = person.get('middle_name', '').strip() if person.get('middle_name') else "" + surname = person.get('surname', '').strip() if person.get('surname') else "" + birthname = person.get('birthname', '').strip() if person.get('birthname') else "" + + if not firstname or not surname: + continue + + name_parts = { + 'firstname': { + 'orig': firstname, + 'norm': normalize_text(firstname) if firstname else "", + 'init': normalize_text(firstname)[0] if firstname else "" + }, + 'surname': { + 'orig': surname, + 'norm': normalize_text(surname) if surname else "", + 'init': normalize_text(surname)[0] if surname else "" + } + } + + if middle_name: + name_parts['middle_name'] = { + 'orig': middle_name, + 'norm': normalize_text(middle_name) if middle_name else "", + 'init': normalize_text(middle_name)[0] if middle_name else "" + } + + if birthname and normalize_text(birthname) != normalize_text(surname): + name_parts['birthname'] = { + 'orig': birthname, + 'norm': normalize_text(birthname), + 'init': normalize_text(birthname)[0] if birthname else "" + } + + person_patterns = set() + + def create_pattern(parts, formats, separators=None): + if separators is None: + separators = [""] + + patterns = [] + for fmt in formats: + for sep in separators: + pattern_parts = [] + for part_type, part_name in fmt: + if part_name in parts and part_type in parts[part_name]: + pattern_parts.append(re.escape(parts[part_name][part_type])) + if pattern_parts: + patterns.append(r"\b" + sep.join(pattern_parts) + r"\b") + return patterns + + name_formats = [ + [('orig', 'firstname'), ('orig', 'surname')], + [('norm', 'firstname'), ('norm', 'surname')], + [('orig', 'surname'), ('orig', 'firstname')], + [('norm', 'surname'), ('norm', 'firstname')], + ] + if 'middle_name' in name_parts: + name_formats = [ + [('orig', 'firstname'), ('orig', 'middle_name'), ('orig', 'surname')], + [('norm', 'firstname'), ('norm', 'middle_name'), ('norm', 'surname')], + ] + + person_patterns.update(create_pattern(name_parts, name_formats, [" ", ""])) + + if 'middle_name' in name_parts: + middle_name_formats = [ + [('orig', 'firstname'), ('orig', 'middle_name')], + [('norm', 'firstname'), ('norm', 'middle_name')], + [('orig', 'middle_name'), ('orig', 'surname')], + [('norm', 'middle_name'), ('norm', 'surname')], + ] + person_patterns.update(create_pattern(name_parts, middle_name_formats, [" ", ""])) + + if 'birthname' in name_parts and name_parts['surname']['orig'] != name_parts['birthname']['orig']: + birthname_formats = [ + [('orig', 'firstname'), ('orig', 'birthname')], + [('norm', 'firstname'), ('norm', 'birthname')], + [('orig', 'birthname'), ('orig', 'firstname')], + [('norm', 'birthname'), ('norm', 'firstname')], + ] + person_patterns.update(create_pattern(name_parts, birthname_formats, [" ", ""])) + + initial_formats = [ + [('init', 'firstname'), ('init', 'middle_name'), ('init', 'surname')], + [('init', 'firstname'), ('init', 'surname')], + ] + person_patterns.update(create_pattern(name_parts, initial_formats, ["", ".", " ", ". "])) + + if 'middle_name' in name_parts: + triple_initial_formats = [ + [('init', 'firstname'), ('init', 'middle_name'), ('init', 'surname')], + ] + person_patterns.update(create_pattern(name_parts, triple_initial_formats, ["", ".", " ", ". "])) + + compiled_patterns = [re.compile(pattern, re.IGNORECASE) for pattern in person_patterns] + + patterns_dict[person_id] = compiled_patterns + + return patterns_dict + + +def extract_person_name_with_regex(found_dict, process_comment, patterns_dict, people): + cleaned_text = process_comment + all_matches = [] + + for person_id, patterns in patterns_dict.items(): + person = next((p for p in people if p.get('id') == person_id), None) + if not person: + continue + + firstname_norm = normalize_text(person.get("firstname", "").strip()) if person.get("firstname") else "" + middle_name_norm = normalize_text(person.get("middle_name", "").strip()) if person.get("middle_name") else "" + surname_norm = normalize_text(person.get("surname", "").strip()) if person.get("surname") else "" + birthname_norm = normalize_text(person.get("birthname", "").strip()) if person.get("birthname") else "" + + text_norm = normalize_text(process_comment) + person_matches = [] + + for pattern in patterns: + for match in pattern.finditer(text_norm): + start, end = match.span() + matched_text = process_comment[start:end] + matched_text_norm = normalize_text(matched_text) + + is_valid_match = False + + # Strict validation: require both firstname AND surname/birthname + # No single-word matches allowed + if len(matched_text_norm.split()) <= 1: + # Single word matches are not allowed + is_valid_match = False + else: + # For multi-word matches, require firstname AND (surname OR birthname) + has_firstname = firstname_norm and firstname_norm in matched_text_norm + has_surname = surname_norm and surname_norm in matched_text_norm + has_birthname = birthname_norm and birthname_norm in matched_text_norm + + # Both firstname and surname/birthname must be present + if (has_firstname and has_surname) or (has_firstname and has_birthname): + is_valid_match = True + + if is_valid_match: + person_matches.append({ + 'matched_text': matched_text, + 'start': start, + 'end': end + }) + + if person_matches: + person_matches.sort(key=lambda x: len(x['matched_text']), reverse=True) + + non_overlapping_matches = [] + for match in person_matches: + overlaps = False + for existing_match in non_overlapping_matches: + if (match['start'] < existing_match['end'] and match['end'] > existing_match['start']): + overlaps = True + break + + if not overlaps: + non_overlapping_matches.append(match) + + if non_overlapping_matches: + found_dict["name_match"] = person + all_matches.extend([(match, person) for match in non_overlapping_matches]) + + if all_matches: + all_matches.sort(key=lambda x: x[0]['start'], reverse=True) + + for match, person in all_matches: + matched_text = match['matched_text'] + + matched_words = matched_text.split() + + for word in matched_words: + word_norm = normalize_text(word).strip() + + if not word_norm: + continue + + text_norm = normalize_text(cleaned_text) + for word_match in re.finditer(rf'\b{re.escape(word_norm)}\b', text_norm, re.IGNORECASE): + start, end = word_match.span() + cleaned_text = cleaned_text[:start] + ' ' * (end - start) + cleaned_text[end:] + + cleaned_text = re.sub(r'\s+', ' ', cleaned_text).strip() + + return found_dict, cleaned_text + + +def extract_build_parts_info(found_dict, process_comment): + + """ + Regex of parts such as : + 2 nolu daire + 9 NUMARALI DAI + daire 3 + 3 nolu dairenin + 11nolu daire + Daire No 12 + 2NOLU DAIRE + 12 No lu daire + D:10 + NO:11 + NO :3 + """ + # Initialize apartment number variable + apartment_number = None + cleaned_text = process_comment + + def clean_text_apartment_number(text, match): + clean_text = text.replace(match.group(0), '').strip() + clean_text = re.sub(r'\s+', ' ', clean_text).strip() + return clean_text + + # Pattern 1: X nolu daire (with space) + pattern1 = re.compile(r'(\d+)\s*nolu\s*daire', re.IGNORECASE) + match = pattern1.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 4: X nolu dairenin + pattern4 = re.compile(r'(\d+)\s*nolu\s*daire\w*', re.IGNORECASE) + match = pattern4.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 5: XNolu daire (without space) + pattern5 = re.compile(r'(\d+)nolu\s*daire', re.IGNORECASE) + match = pattern5.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 7: XNOLU DAIRE (all caps, no space) + pattern7 = re.compile(r'(\d+)nolu\s*daire', re.IGNORECASE) + match = pattern7.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 8: X No lu daire + pattern8 = re.compile(r'(\d+)\s*no\s*lu\s*daire', re.IGNORECASE) + match = pattern8.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 6: Daire No X + pattern6 = re.compile(r'daire\s*no\s*(\d+)', re.IGNORECASE) + match = pattern6.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 2: X NUMARALI DAI + pattern2 = re.compile(r'(\d+)\s*numarali\s*dai', re.IGNORECASE) + match = pattern2.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 3: daire X + pattern3 = re.compile(r'daire\s*(\d+)', re.IGNORECASE) + match = pattern3.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 9: D:X + pattern9 = re.compile(r'd\s*:\s*(\d+)', re.IGNORECASE) + match = pattern9.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + # Pattern 10: NO:X or NO :X + pattern10 = re.compile(r'no\s*:\s*(\d+)', re.IGNORECASE) + match = pattern10.search(cleaned_text) + if match: + apartment_number = match.group(1) + found_dict['apartment_number'] = apartment_number + return found_dict, clean_text_apartment_number(cleaned_text, match) + + return found_dict, cleaned_text + + +def extract_months(found_dict, process_comment): + """ + Extract Turkish month names and abbreviations from the process comment + """ + original_text = process_comment + # Updated dictionary with normalized keys for better matching + month_to_number_dict = { + "ocak": 1, "şubat": 2, "mart": 3, "nisan": 4, "mayıs": 5, "haziran": 6, + "temmuz": 7, "ağustos": 8, "eylül": 9, "ekim": 10, "kasım": 11, "aralık": 12, + # Add normalized versions without Turkish characters + "ocak": 1, "subat": 2, "mart": 3, "nisan": 4, "mayis": 5, "haziran": 6, + "temmuz": 7, "agustos": 8, "eylul": 9, "ekim": 10, "kasim": 11, "aralik": 12 + } + + def clean_text_month(text, match): + clean_text = text.replace(match.group(0), '').strip() + clean_text = re.sub(r'\s+', ' ', clean_text).strip() + return clean_text + + def normalize_turkish(text): + """Properly normalize Turkish text for case-insensitive comparison""" + text = text.lower() + text = text.replace('i̇', 'i') # Handle dotted i properly + text = text.replace('ı', 'i') # Convert dotless i to regular i for matching + text = unidecode(text) # Remove other diacritics + return text + + if 'months' not in found_dict: + found_dict['months'] = [] + + months_found, working_text = False, original_text + + for month in turkish_months: + pattern = re.compile(r'\b' + re.escape(month) + r'\b', re.IGNORECASE) + for match in pattern.finditer(original_text): + matched_text = match.group(0) + + normalized_month = normalize_turkish(month) + month_number = None + + if month.lower() in month_to_number_dict: + month_number = month_to_number_dict[month.lower()] + elif normalized_month in month_to_number_dict: + month_number = month_to_number_dict[normalized_month] + + month_info = {'name': month, 'number': month_number} + found_dict['months'].append(month_info) + months_found = True + working_text = working_text.replace(matched_text, '', 1) + + for abbr, full_month in turkish_months_abbr.items(): + pattern = re.compile(r'\b' + re.escape(abbr) + r'\b', re.IGNORECASE) + + for match in pattern.finditer(working_text): + matched_text = match.group(0) + normalized_month = normalize_turkish(full_month) + month_number = None + + if full_month.lower() in month_to_number_dict: + month_number = month_to_number_dict[full_month.lower()] + elif normalized_month in month_to_number_dict: + month_number = month_to_number_dict[normalized_month] + + month_info = {'name': full_month, 'number': month_number} + found_dict['months'].append(month_info) + months_found = True + working_text = working_text.replace(matched_text, '', 1) + + return found_dict, working_text + + +def extract_year(found_dict, process_comment): + """ + Extract years from the process comment + """ + original_text = process_comment + + if 'years' not in found_dict: + found_dict['years'] = [] + + working_text = original_text + + for year in range(start_year, current_year + 1): + pattern = re.compile(r'\b' + str(year) + r'\b', re.IGNORECASE) + for match in pattern.finditer(original_text): + matched_text = match.group(0) + if str(matched_text).isdigit(): + found_dict['years'].append(int(matched_text)) + working_text = working_text.replace(matched_text, '', 1) + + return found_dict, working_text + + +def extract_payment_type(found_dict, process_comment): + """ + Extract payment type from the process comment + aidat + AİD + aidatı + TADİLAT + YAKIT + yakıt + yakit + """ + original_text = process_comment + working_text = original_text + + if 'payment_types' not in found_dict: + found_dict['payment_types'] = [] + + payment_keywords = { + 'aidat': ['aidat', 'aİd', 'aid', 'aidatı', 'aidati'], + 'tadilat': ['tadilat', 'tadİlat', 'tadilatı'], + 'yakit': ['yakit', 'yakıt', 'yakıtı', 'yakiti'] + } + + for payment_type, keywords in payment_keywords.items(): + for keyword in keywords: + pattern = re.compile(r'\b' + keyword + r'\b', re.IGNORECASE) + for match in pattern.finditer(original_text): + matched_text = match.group(0) + if payment_type not in found_dict['payment_types']: + found_dict['payment_types'].append(payment_type) + working_text = working_text.replace(matched_text, '', 1) + + return found_dict, working_text + + +def main(session, account_records, people): + + list_of_regex_patterns = generate_dictonary_of_patterns(people=people) + dicts_found = dict() + dicts_not_found = dict() + count_extracted = 0 + for account_record in account_records: + account_record_id = str(account_record["id"]) + found_dict = {} + process_comment_iteration = clean_text(text=account_record["process_comment"]) + found_dict, cleaned_process_comment = extract_person_name_with_regex( + found_dict=found_dict, process_comment=process_comment_iteration, patterns_dict=list_of_regex_patterns, people=people + ) + + found_dict, cleaned_process_comment = extract_build_parts_info( + found_dict=found_dict, process_comment=cleaned_process_comment + ) + found_dict, cleaned_process_comment = extract_months( + found_dict=found_dict, process_comment=cleaned_process_comment + ) + found_dict, cleaned_process_comment = extract_year( + found_dict=found_dict, process_comment=cleaned_process_comment + ) + found_dict, cleaned_process_comment = extract_payment_type( + found_dict=found_dict, process_comment=cleaned_process_comment + ) + if found_dict: + dicts_found[str(account_record_id)] = found_dict + else: + dicts_not_found[str(account_record_id)] = account_record_id + + + for id_, item in dicts_found.items(): + AccountRecordsPredict.set_session(session) + AccountRecords.set_session(session) + + months_are_valid = bool(item.get("months", [])) + years_are_valid = bool(item.get("years", [])) + payment_types_are_valid = bool(item.get("payment_types", [])) + apartment_number_are_valid = bool(item.get("apartment_number", [])) + person_name_are_valid = bool(item.get("name_match", [])) + account_record_to_save = AccountRecords.query.filter_by(id=int(id_)).first() + save_dict = dict( + account_records_id=account_record_to_save.id, account_records_uu_id=str(account_record_to_save.uu_id), prediction_model="regex", treshold=1, is_first_prediction=False + ) + update_dict = dict(prediction_model="regex", treshold=1, is_first_prediction=False) + if any([months_are_valid, years_are_valid, payment_types_are_valid, apartment_number_are_valid, person_name_are_valid]): + count_extracted += 1 + if months_are_valid: + print(f"months: {item['months']}") + data_to_save = dumps({"data": item['months']}) + prediction_result = AccountRecordsPredict.query.filter_by(account_records_id=account_record_to_save.id, prediction_field="months", prediction_model="regex").first() + if not prediction_result: + created_account_prediction = AccountRecordsPredict.create(**save_dict, prediction_field="months", prediction_result=data_to_save) + created_account_prediction.save() + else: + prediction_result.update(**update_dict, prediction_result=data_to_save) + prediction_result.save() + if years_are_valid: + print(f"years: {item['years']}") + data_to_save = dumps({"data": item['years']}) + prediction_result = AccountRecordsPredict.query.filter_by(account_records_id=account_record_to_save.id, prediction_field="years", prediction_model="regex").first() + if not prediction_result: + created_account_prediction = AccountRecordsPredict.create(**save_dict, prediction_field="years", prediction_result=data_to_save) + created_account_prediction.save() + else: + prediction_result.update(**update_dict, prediction_result=data_to_save) + prediction_result.save() + if payment_types_are_valid: + print(f"payment_types: {item['payment_types']}") + data_to_save = dumps({"data": item['payment_types']}) + prediction_result = AccountRecordsPredict.query.filter_by(account_records_id=account_record_to_save.id, prediction_field="payment_types", prediction_model="regex").first() + if not prediction_result: + created_account_prediction = AccountRecordsPredict.create(**save_dict, prediction_field="payment_types", prediction_result=data_to_save) + created_account_prediction.save() + else: + prediction_result.update(**update_dict, prediction_result=data_to_save) + prediction_result.save() + if apartment_number_are_valid: + print(f"apartment_number: {item['apartment_number']}") + prediction_result = AccountRecordsPredict.query.filter_by(account_records_id=account_record_to_save.id, prediction_field="apartment_number", prediction_model="regex").first() + if not prediction_result: + created_account_prediction = AccountRecordsPredict.create(**save_dict, prediction_field="apartment_number", prediction_result=item['apartment_number']) + created_account_prediction.save() + else: + prediction_result.update(**update_dict, prediction_result=item['apartment_number']) + prediction_result.save() + if person_name_are_valid: + print(f"person_name: {item['name_match']}") + data_to_save = dumps({"data": item['name_match']}) + prediction_result = AccountRecordsPredict.query.filter_by(account_records_id=account_record_to_save.id, prediction_field="person_name", prediction_model="regex").first() + if not prediction_result: + created_account_prediction = AccountRecordsPredict.create(**save_dict, prediction_field="person_name", prediction_result=data_to_save) + created_account_prediction.save() + else: + prediction_result.update(**update_dict, prediction_result=data_to_save) + prediction_result.save() + + print("\n===== SUMMARY =====") + print(f"extracted data total : {count_extracted}") + print(f"not extracted data total : {len(account_records) - count_extracted}") + print(f"Total account records processed : {len(account_records)}") + + +if __name__ == "__main__": + + session_factory = get_session_factory() + session = session_factory() + + turkish_months = ["OCAK", "ŞUBAT", "MART", "NİSAN", "MAYIS", "HAZİRAN", "TEMMUZ", "AĞUSTOS", "EYLÜL", "EKİM", "KASIM", "ARALIK"] + turkish_months_abbr = { + "OCA": "OCAK", "SUB": "ŞUBAT", "ŞUB": "ŞUBAT", "MAR": "MART", "NIS": "NİSAN", "MAY": "MAYIS", "HAZ": "HAZİRAN", "HZR": "HAZİRAN", + "TEM": "TEMMUZ", "AGU": "AĞUSTOS", "AGT": "AĞUSTOS", "EYL": "EYLÜL", "EKI": "EKİM", "KAS": "KASIM", "ARA": "ARALIK", + } + start_year = 1950 + current_year = arrow.now().year + + people_query = sqlalchemy_text(""" + SELECT DISTINCT ON (p.id) p.firstname, p.middle_name, p.surname, p.birthname, bl.id + FROM public.people as p + INNER JOIN public.build_living_space as bl ON bl.person_id = p.id + INNER JOIN public.build_parts as bp ON bp.id = bl.build_parts_id + INNER JOIN public.build as b ON b.id = bp.build_id + WHERE b.id = 1 + ORDER BY p.id + """) + + people_raw = session.execute(people_query).all() + remove_duplicate = list() + clean_people_list = list() + for person in people_raw: + merged_name = f"{person[0]} {person[1]} {person[2]} {person[3]}" + if merged_name not in remove_duplicate: + clean_people_list.append(person) + remove_duplicate.append(merged_name) + + people = [{"firstname": p[0], "middle_name": p[1], "surname": p[2], "birthname": p[3], 'id': p[4]} for p in clean_people_list] + query_account_records = sqlalchemy_text(""" + SELECT a.id, a.iban, a.bank_date, a.process_comment FROM public.account_records as a where currency_value > 0 + """) # and bank_date::date >= '2020-01-01' + account_records = session.execute(query_account_records).all() + account_records = [{"id": ar[0], "iban": ar[1], "bank_date": ar[2], "process_comment": ar[3]} for ar in account_records] + + try: + main(session=session, account_records=account_records, people=people) + except Exception as e: + print(f"{e}") + + session.close() + session_factory.remove() diff --git a/ServicesBank/RoutineEmail/Dockerfile b/ServicesBank/RoutineEmail/Dockerfile new file mode 100644 index 0000000..c6ff012 --- /dev/null +++ b/ServicesBank/RoutineEmail/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim + +WORKDIR / + +RUN apt-get update && apt-get install -y --no-install-recommends gcc && rm -rf /var/lib/apt/lists/* && pip install --no-cache-dir poetry + +COPY /ServicesBank/RoutineEmail/pyproject.toml ./pyproject.toml + +RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main && pip cache purge && rm -rf ~/.cache/pypoetry +RUN apt-get update && apt-get install -y cron + +COPY /ServicesBank/RoutineEmail / + +RUN chmod +x /run_app.sh + +COPY /ServicesBank/RoutineEmail / +COPY /ServicesApi/Controllers /ServicesApi/Controllers +COPY /ServicesBank/Depends/config.py /ServicesBank/Depends/config.py +COPY /ServicesBank/Depends/template_accounts.html /template_accounts.html + +ENV PYTHONPATH=/ PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 + +RUN touch /var/log/cron.log +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ServicesBank/RoutineEmail/README.md b/ServicesBank/RoutineEmail/README.md new file mode 100644 index 0000000..1e72d3f --- /dev/null +++ b/ServicesBank/RoutineEmail/README.md @@ -0,0 +1,69 @@ +# Routine Email Service + +## Overview +This service sends automated email reports about account records at scheduled times using cron. It retrieves account records from a PostgreSQL database, formats them into an HTML email, and sends them to specified recipients. + +## Environment Setup +The service requires the following environment variables: + +### Email Configuration +- `EMAIL_HOST`: SMTP server address (e.g., "10.10.2.34") +- `EMAIL_USERNAME`: Email sender address (e.g., "example@domain.com") +- `EMAIL_PASSWORD`: Email password (sensitive) +- `EMAIL_PORT`: SMTP port (e.g., 587) +- `EMAIL_SEND`: Flag to enable/disable email sending (1 = enabled) + +### Database Configuration +- `DB_HOST`: PostgreSQL server address (e.g., "10.10.2.14") +- `DB_USER`: Database username (e.g., "postgres") +- `DB_PASSWORD`: Database password (sensitive) +- `DB_PORT`: Database port (e.g., 5432) +- `DB_NAME`: Database name (e.g., "postgres") + +## Cron Job Configuration +The service is configured to run daily at 11:00 Istanbul Time (08:00 UTC). This is set up in the entrypoint.sh script. + +## Docker Container Setup + +### Key Files + +1. **Dockerfile**: Defines the container image with Python and cron + +2. **entrypoint.sh**: Container entrypoint script that: + - Creates an environment file (/env.sh) with all configuration variables + - Sets up the crontab to run run_app.sh at the scheduled time + - Starts the cron service + - Tails the log file for monitoring + +3. **run_app.sh**: Script executed by cron that: + - Sources the environment file to get all configuration + - Exports variables to make them available to the Python script + - Runs the Python application + - Logs environment and execution results + +### Environment Variable Handling +Cron jobs run with a minimal environment that doesn't automatically include Docker container environment variables. Our solution: + +1. Captures all environment variables from Docker to a file at container startup +2. Has the run_app.sh script source this file before execution +3. Explicitly exports all variables to ensure they're available to the Python script + +## Logs +Logs are written to `/var/log/cron.log` and can be viewed with: +```bash +docker exec routine_email_service tail -f /var/log/cron.log +``` + +## Manual Execution +To run the service manually: +```bash +docker exec routine_email_service /run_app.sh +``` + +## Docker Compose Configuration +In the docker-compose.yml file, the service needs an explicit entrypoint configuration: +```yaml +entrypoint: ["/entrypoint.sh"] +``` + +This ensures the entrypoint script runs when the container starts. diff --git a/ServicesBank/RoutineEmail/app.py b/ServicesBank/RoutineEmail/app.py new file mode 100644 index 0000000..9aabf8f --- /dev/null +++ b/ServicesBank/RoutineEmail/app.py @@ -0,0 +1,157 @@ +import arrow +from typing import List, Any + +from Schemas import AccountRecords +from jinja2 import Environment, FileSystemLoader +from Controllers.Email.send_email import EmailSendModel, EmailService + + +def render_email_template( + headers: List[str], rows: List[List[Any]], balance_error: bool, bank_balance: str +) -> str: + """ + Render the HTML email template with the provided data. + + Args: + headers: List of column headers for the table + rows: List of data rows for the table + balance_error: Flag indicating if there's a balance discrepancy + bank_balance: Current bank balance formatted as string + + Returns: + Rendered HTML template as string + """ + try: + # Look for template in ServiceDepends directory + env = Environment(loader=FileSystemLoader("/")) + template = env.get_template("template_accounts.html") + + # Render template with variables + return template.render( + headers=headers, + rows=rows, + bank_balance=bank_balance, + balance_error=balance_error, + today=str(arrow.now().date()), + ) + except Exception as e: + print("Exception render template:", e) + raise + + +def send_email_to_given_address(send_to: str, html_template: str) -> bool: + """ + Send email with the rendered HTML template to the specified address. + + Args: + send_to: Email address of the recipient + html_template: Rendered HTML template content + + Returns: + Boolean indicating if the email was sent successfully + """ + today = arrow.now() + subject = f"{str(today.date())} Gunes Apt. Cari Durum Bilgilendirme Raporu" + + # Create email parameters using EmailSendModel + email_params = EmailSendModel( + subject=subject, + html=html_template, + receivers=[send_to], + text=f"Gunes Apt. Cari Durum Bilgilendirme Raporu - {today.date()}", + ) + + try: + # Use the context manager to handle connection errors + with EmailService.new_session() as email_session: + # Send email through the service + return EmailService.send_email(email_session, email_params) + except Exception as e: + print(f"Exception send email: {e}") + return False + + +def set_account_records_to_send_email() -> bool: + """ + Retrieve account records from the database, format them, and send an email report. + + Usage: + from app import set_account_records_to_send_email + + Returns: + Boolean indicating if the process completed successfully + """ + # Get database session and retrieve records + with AccountRecords.new_session() as db_session: + account_records_query = AccountRecords.filter_all(db=db_session).query + + # Get the 3 most recent records + account_records: List[AccountRecords] = ( + account_records_query.order_by( + AccountRecords.bank_date.desc(), + AccountRecords.iban.desc(), + ) + .limit(3) + .all() + ) + + # Check if we have enough records + if len(account_records) < 2: + print(f"Not enough records found: {len(account_records)}") + return False + + # Check for balance discrepancy + first_record, second_record = account_records[0], account_records[1] + expected_second_balance = ( + first_record.bank_balance - first_record.currency_value + ) + balance_error = expected_second_balance != second_record.bank_balance + + if balance_error: + print( + f"Balance error detected {expected_second_balance} != {second_record.bank_balance}" + ) + + # Format rows for the email template + list_of_rows = [] + for record in account_records: + list_of_rows.append( + [ + record.bank_date.strftime("%d/%m/%Y %H:%M"), + record.process_comment, + f"{record.currency_value:,.2f}", + f"{record.bank_balance:,.2f}", + ] + ) + # Get the most recent bank balance + last_bank_balance = sorted( + account_records, key=lambda x: x.bank_date, reverse=True + )[0].bank_balance + # Define headers for the table + headers = [ + "Ulaştığı Tarih", + "Banka Transaksiyonu Ek Bilgi", + "Aktarım Değeri", + "Banka Bakiyesi", + ] + + # Recipient email address + send_to = "karatay@mehmetkaratay.com.tr" + + # Render email template + html_template = render_email_template( + headers=headers, + rows=list_of_rows, + balance_error=balance_error, + bank_balance=f"{last_bank_balance:,.2f}", + ) + + # Send the email + return send_email_to_given_address(send_to=send_to, html_template=html_template) + + +if __name__ == "__main__": + success = set_account_records_to_send_email() + print("Email sent successfully" if success else "Failed to send email") + exit_code = 0 if success else 1 + exit(exit_code) diff --git a/ServicesBank/RoutineEmail/entrypoint.sh b/ServicesBank/RoutineEmail/entrypoint.sh new file mode 100644 index 0000000..d1bc6ca --- /dev/null +++ b/ServicesBank/RoutineEmail/entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Create environment file that will be available to cron jobs +echo "# Environment variables for cron jobs" > /env.sh +echo "EMAIL_HOST=\"$EMAIL_HOST\"" >> /env.sh +echo "EMAIL_USERNAME=\"$EMAIL_USERNAME\"" >> /env.sh +echo "EMAIL_PASSWORD=\"$EMAIL_PASSWORD\"" >> /env.sh +echo "EMAIL_PORT=$EMAIL_PORT" >> /env.sh +echo "EMAIL_SEND=$EMAIL_SEND" >> /env.sh +echo "DB_HOST=\"$DB_HOST\"" >> /env.sh +echo "DB_USER=\"$DB_USER\"" >> /env.sh +echo "DB_PASSWORD=\"$DB_PASSWORD\"" >> /env.sh +echo "DB_PORT=$DB_PORT" >> /env.sh +echo "DB_NAME=\"$DB_NAME\"" >> /env.sh + +# Add Python environment variables +echo "PYTHONPATH=/" >> /env.sh +echo "PYTHONUNBUFFERED=1" >> /env.sh +echo "PYTHONDONTWRITEBYTECODE=1" >> /env.sh + +# Make the environment file available to cron +echo "0 8 * * * /run_app.sh >> /var/log/cron.log 2>&1" > /tmp/crontab_list +crontab /tmp/crontab_list + +# Start cron +cron + +# Tail the log file +tail -f /var/log/cron.log diff --git a/ServicesBank/RoutineEmail/pyproject.toml b/ServicesBank/RoutineEmail/pyproject.toml new file mode 100644 index 0000000..ddb7a04 --- /dev/null +++ b/ServicesBank/RoutineEmail/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "routineemailservice" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "arrow>=1.3.0", + "redbox>=0.2.1", + "redis>=5.2.1", + "pydantic-settings>=2.8.1", + "sqlalchemy-mixins>=2.0.5", + "fastapi>=0.115.11", + "jinja2>=3.1.6", + "psycopg2-binary>=2.9.10", + "redmail>=0.6.0", +] diff --git a/ServicesBank/RoutineEmail/run_app.sh b/ServicesBank/RoutineEmail/run_app.sh new file mode 100644 index 0000000..4e54fd4 --- /dev/null +++ b/ServicesBank/RoutineEmail/run_app.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Source the environment file directly +. /env.sh + +# Re-export all variables to ensure they're available to the Python script +export EMAIL_HOST +export EMAIL_USERNAME +export EMAIL_PASSWORD +export EMAIL_PORT +export EMAIL_SEND +export DB_HOST +export DB_USER +export DB_PASSWORD +export DB_PORT +export DB_NAME + +# Python environment variables +export PYTHONPATH +export PYTHONUNBUFFERED +export PYTHONDONTWRITEBYTECODE + +env >> /var/log/cron.log +/usr/local/bin/python /app.py \ No newline at end of file diff --git a/ServicesBank/RoutineEmail/templates/a.txt b/ServicesBank/RoutineEmail/templates/a.txt new file mode 100644 index 0000000..e69de29 diff --git a/bank-services-docker-compose copy.yml b/bank-services-docker-compose copy.yml new file mode 100644 index 0000000..5aade32 --- /dev/null +++ b/bank-services-docker-compose copy.yml @@ -0,0 +1,161 @@ +services: + email_service: + container_name: email_service + build: + context: . + dockerfile: ServicesBank/Email/Dockerfile + networks: + - bank-services-network + environment: + - MAILBOX=bilgilendirme@ileti.isbank.com.tr + - MAIN_MAIL=karatay.berkay@gmail.com + - INFO_MAIL=mehmet.karatay@hotmail.com + - EMAIL_HOST=10.10.2.34 + - EMAIL_USERNAME=isbank@mehmetkaratay.com.tr + - EMAIL_PASSWORD=system + - EMAIL_PORT=993 + - EMAIL_SEND_PORT=587 + - EMAIL_SLEEP=60 + - AUTHORIZE_IBAN=4245-0093333 + - REDIS_HOST=10.10.2.15 + - REDIS_PORT=6379 + - REDIS_PASSWORD=your_strong_password_here + restart: unless-stopped + volumes: + - tempory-email-service:/tmp + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + + finder_build_extractor: + container_name: finder_build_extractor + env_file: + - api_env.env + build: + context: . + dockerfile: ServicesBank/Finder/BuildExtractor/Dockerfile + networks: + - bank-services-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + cpus: 0.25 + mem_limit: 512m + + finder_payment_service: + container_name: finder_payment_service + env_file: + - api_env.env + build: + context: . + dockerfile: ServicesBank/Finder/Payment/Dockerfile + networks: + - bank-services-network + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + cpus: 0.25 + mem_limit: 512m + + parser_service: + container_name: parser_service + build: + context: . + dockerfile: ServicesBank/Parser/Dockerfile + networks: + - bank-services-network + env_file: + - api_env.env + restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + + # writer_service: + # container_name: writer_service + # build: + # context: . + # dockerfile: ServicesBank/WriterService/Dockerfile + # networks: + # - bank-services-network + # environment: + # - REDIS_HOST=10.10.2.15 + # - REDIS_PORT=6379 + # - REDIS_PASSWORD=your_strong_password_here + # - DB_HOST=10.10.2.14 + # - DB_PORT=5432 + # - DB_USER=postgres + # - DB_PASSWORD=password + # - DB_NAME=postgres + # restart: unless-stopped + # logging: + # driver: "json-file" + # options: + # max-size: "10m" + # max-file: "3" + + routine_email_service: + container_name: routine_email_service + build: + context: . + dockerfile: ServicesBank/RoutineEmailService/Dockerfile + entrypoint: ["/entrypoint.sh"] + networks: + - bank-services-network + environment: + - EMAIL_HOST=10.10.2.34 + - EMAIL_USERNAME=karatay@mehmetkaratay.com.tr + - EMAIL_PASSWORD=system + - EMAIL_PORT=587 + - EMAIL_SEND=1 + - DB_HOST=10.10.2.14 + - DB_PORT=5432 + - DB_USER=postgres + - DB_PASSWORD=password + - DB_NAME=postgres + restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + + # sender_service: + # container_name: sender_service + # build: + # context: . + # dockerfile: ServicesBank/SenderService/Dockerfile + # networks: + # - bank-services-network + # environment: + # - EMAIL_HOST=10.10.2.34 + # - EMAIL_USERNAME=karatay@mehmetkaratay.com.tr + # - EMAIL_PASSWORD=system + # - EMAIL_PORT=587 + # - EMAIL_SEND=1 + # - DB_HOST=10.10.2.14 + # - DB_PORT=5432 + # - DB_USER=postgres + # - DB_PASSWORD=password + # - DB_NAME=postgres + # restart: unless-stopped + # logging: + # driver: "json-file" + # options: + # max-size: "10m" + # max-file: "3" + +networks: + bank-services-network: + driver: bridge + +volumes: + tempory-email-service: diff --git a/docker-compose.yml b/docker-compose.yml index e90d757..a8d3ff9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -189,76 +189,45 @@ services: - "8003:8003" # restart: unless-stopped python3 app_accounts.py - finder_build_from_iban_service: - container_name: finder_build_from_iban_service - env_file: - - api_env.env - build: - context: . - dockerfile: ServicesBank/Finder/BuildFromIban/Dockerfile - networks: - - wag-services - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" + # finder_build_living_space_service: + # container_name: finder_build_living_space_service + # env_file: + # - api_env.env + # build: + # context: . + # dockerfile: ServicesBank/Finder/BuildLivingSpace/Dockerfile + # networks: + # - wag-services + # logging: + # driver: "json-file" + # options: + # max-size: "10m" + # max-file: "3" - finder_build_living_space_service: - container_name: finder_build_living_space_service - env_file: - - api_env.env - build: - context: . - dockerfile: ServicesBank/Finder/BuildLivingSpace/Dockerfile - networks: - - wag-services - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" + # finder_decision_book_service: + # container_name: finder_decision_book_service + # env_file: + # - api_env.env + # build: + # context: . + # dockerfile: ServicesBank/Finder/DecisionBook/Dockerfile + # networks: + # - wag-services + # logging: + # driver: "json-file" + # options: + # max-size: "10m" + # max-file: "3" - finder_decision_book_service: - container_name: finder_decision_book_service - env_file: - - api_env.env - build: - context: . - dockerfile: ServicesBank/Finder/DecisionBook/Dockerfile - networks: - - wag-services - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" + # zemberek-api: + # build: + # context: . + # dockerfile: ServicesBank/Zemberek/Dockerfile + # container_name: zemberek-api + # ports: + # - "8111:8111" + # restart: unless-stopped - zemberek-api: - build: - context: . - dockerfile: ServicesBank/Zemberek/Dockerfile - container_name: zemberek-api - ports: - - "8111:8111" - restart: unless-stopped - - finder_payment_service: - container_name: finder_payment_service - env_file: - - api_env.env - build: - context: . - dockerfile: ServicesBank/Finder/Payment/Dockerfile - networks: - - wag-services - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "3" - # cpus: 0.25 - # mem_limit: 512m # address_service: # container_name: address_service @@ -354,7 +323,7 @@ services: context: . dockerfile: ServicesApi/Builds/Initial/Dockerfile environment: - - SET_ALEMBIC=0 + - SET_ALEMBIC=1 networks: - wag-services env_file: diff --git a/ServicesBank/Finder/DecisionBook/.dockerignore b/trash/XBuildFromIban/.dockerignore similarity index 100% rename from ServicesBank/Finder/DecisionBook/.dockerignore rename to trash/XBuildFromIban/.dockerignore diff --git a/ServicesBank/Finder/BuildFromIban/Dockerfile b/trash/XBuildFromIban/Dockerfile similarity index 100% rename from ServicesBank/Finder/BuildFromIban/Dockerfile rename to trash/XBuildFromIban/Dockerfile diff --git a/ServicesBank/Finder/BuildFromIban/README.md b/trash/XBuildFromIban/README.md similarity index 100% rename from ServicesBank/Finder/BuildFromIban/README.md rename to trash/XBuildFromIban/README.md diff --git a/ServicesBank/Finder/BuildFromIban/entrypoint.sh b/trash/XBuildFromIban/entrypoint.sh similarity index 100% rename from ServicesBank/Finder/BuildFromIban/entrypoint.sh rename to trash/XBuildFromIban/entrypoint.sh diff --git a/trash/XBuildFromIban/run_app.sh b/trash/XBuildFromIban/run_app.sh new file mode 100644 index 0000000..83dcf19 --- /dev/null +++ b/trash/XBuildFromIban/run_app.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Source the environment file directly +. /env.sh + +# Re-export all variables to ensure they're available to the Python script +export POSTGRES_USER +export POSTGRES_PASSWORD +export POSTGRES_DB +export POSTGRES_HOST +export POSTGRES_PORT +export POSTGRES_ENGINE +export POSTGRES_POOL_PRE_PING +export POSTGRES_POOL_SIZE +export POSTGRES_MAX_OVERFLOW +export POSTGRES_POOL_RECYCLE +export POSTGRES_POOL_TIMEOUT +export POSTGRES_ECHO + +# Python environment variables +export PYTHONPATH +export PYTHONUNBUFFERED +export PYTHONDONTWRITEBYTECODE + +# env >> /var/log/cron.log +/usr/local/bin/python /runner.py diff --git a/ServicesBank/Finder/BuildFromIban/runner.py b/trash/XBuildFromIban/runner.py similarity index 100% rename from ServicesBank/Finder/BuildFromIban/runner.py rename to trash/XBuildFromIban/runner.py diff --git a/trash/XBuildLivingSpace/.dockerignore b/trash/XBuildLivingSpace/.dockerignore new file mode 100644 index 0000000..1449c90 --- /dev/null +++ b/trash/XBuildLivingSpace/.dockerignore @@ -0,0 +1,93 @@ +# Git +.git +.gitignore +.gitattributes + + +# CI +.codeclimate.yml +.travis.yml +.taskcluster.yml + +# Docker +docker-compose.yml +service_app/Dockerfile +.docker +.dockerignore + +# Byte-compiled / optimized / DLL files +**/__pycache__/ +**/*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +service_app/env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Virtual environment +service_app/.env +.venv/ +venv/ + +# PyCharm +.idea + +# Python mode for VIM +.ropeproject +**/.ropeproject + +# Vim swap files +**/*.swp + +# VS Code +.vscode/ + +test_application/ + + diff --git a/ServicesBank/Finder/BuildLivingSpace/Dockerfile b/trash/XBuildLivingSpace/Dockerfile similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/Dockerfile rename to trash/XBuildLivingSpace/Dockerfile diff --git a/ServicesBank/Finder/BuildLivingSpace/README.md b/trash/XBuildLivingSpace/README.md similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/README.md rename to trash/XBuildLivingSpace/README.md diff --git a/ServicesBank/Finder/BuildLivingSpace/configs.py b/trash/XBuildLivingSpace/configs.py similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/configs.py rename to trash/XBuildLivingSpace/configs.py diff --git a/trash/XBuildLivingSpace/entrypoint.sh b/trash/XBuildLivingSpace/entrypoint.sh new file mode 100644 index 0000000..73f4e4d --- /dev/null +++ b/trash/XBuildLivingSpace/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Create environment file that will be available to cron jobs +echo "POSTGRES_USER=\"$POSTGRES_USER\"" >> /env.sh +echo "POSTGRES_PASSWORD=\"$POSTGRES_PASSWORD\"" >> /env.sh +echo "POSTGRES_DB=\"$POSTGRES_DB\"" >> /env.sh +echo "POSTGRES_HOST=\"$POSTGRES_HOST\"" >> /env.sh +echo "POSTGRES_PORT=$POSTGRES_PORT" >> /env.sh +echo "POSTGRES_ENGINE=\"$POSTGRES_ENGINE\"" >> /env.sh +echo "POSTGRES_POOL_PRE_PING=\"$POSTGRES_POOL_PRE_PING\"" >> /env.sh +echo "POSTGRES_POOL_SIZE=$POSTGRES_POOL_SIZE" >> /env.sh +echo "POSTGRES_MAX_OVERFLOW=$POSTGRES_MAX_OVERFLOW" >> /env.sh +echo "POSTGRES_POOL_RECYCLE=$POSTGRES_POOL_RECYCLE" >> /env.sh +echo "POSTGRES_POOL_TIMEOUT=$POSTGRES_POOL_TIMEOUT" >> /env.sh +echo "POSTGRES_ECHO=\"$POSTGRES_ECHO\"" >> /env.sh + +# Add Python environment variables +echo "PYTHONPATH=/" >> /env.sh +echo "PYTHONUNBUFFERED=1" >> /env.sh +echo "PYTHONDONTWRITEBYTECODE=1" >> /env.sh + +# Make the environment file available to cron +echo "*/15 * * * * /run_app.sh >> /var/log/cron.log 2>&1" > /tmp/crontab_list +crontab /tmp/crontab_list + +# Start cron +cron + +# Tail the log file +tail -f /var/log/cron.log diff --git a/ServicesBank/Finder/BuildLivingSpace/parser.py b/trash/XBuildLivingSpace/parser.py similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/parser.py rename to trash/XBuildLivingSpace/parser.py diff --git a/ServicesBank/Finder/BuildLivingSpace/regex_func.py b/trash/XBuildLivingSpace/regex_func.py similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/regex_func.py rename to trash/XBuildLivingSpace/regex_func.py diff --git a/ServicesBank/Finder/BuildLivingSpace/run_app.sh b/trash/XBuildLivingSpace/run_app.sh similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/run_app.sh rename to trash/XBuildLivingSpace/run_app.sh diff --git a/ServicesBank/Finder/BuildLivingSpace/runner.py b/trash/XBuildLivingSpace/runner.py similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/runner.py rename to trash/XBuildLivingSpace/runner.py diff --git a/ServicesBank/Finder/BuildLivingSpace/validations.py b/trash/XBuildLivingSpace/validations.py similarity index 100% rename from ServicesBank/Finder/BuildLivingSpace/validations.py rename to trash/XBuildLivingSpace/validations.py diff --git a/trash/XDecisionBook/.dockerignore b/trash/XDecisionBook/.dockerignore new file mode 100644 index 0000000..1449c90 --- /dev/null +++ b/trash/XDecisionBook/.dockerignore @@ -0,0 +1,93 @@ +# Git +.git +.gitignore +.gitattributes + + +# CI +.codeclimate.yml +.travis.yml +.taskcluster.yml + +# Docker +docker-compose.yml +service_app/Dockerfile +.docker +.dockerignore + +# Byte-compiled / optimized / DLL files +**/__pycache__/ +**/*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +service_app/env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Virtual environment +service_app/.env +.venv/ +venv/ + +# PyCharm +.idea + +# Python mode for VIM +.ropeproject +**/.ropeproject + +# Vim swap files +**/*.swp + +# VS Code +.vscode/ + +test_application/ + + diff --git a/ServicesBank/Finder/DecisionBook/Dockerfile b/trash/XDecisionBook/Dockerfile similarity index 100% rename from ServicesBank/Finder/DecisionBook/Dockerfile rename to trash/XDecisionBook/Dockerfile diff --git a/ServicesBank/Finder/DecisionBook/README.md b/trash/XDecisionBook/README.md similarity index 100% rename from ServicesBank/Finder/DecisionBook/README.md rename to trash/XDecisionBook/README.md diff --git a/trash/XDecisionBook/entrypoint.sh b/trash/XDecisionBook/entrypoint.sh new file mode 100644 index 0000000..73f4e4d --- /dev/null +++ b/trash/XDecisionBook/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Create environment file that will be available to cron jobs +echo "POSTGRES_USER=\"$POSTGRES_USER\"" >> /env.sh +echo "POSTGRES_PASSWORD=\"$POSTGRES_PASSWORD\"" >> /env.sh +echo "POSTGRES_DB=\"$POSTGRES_DB\"" >> /env.sh +echo "POSTGRES_HOST=\"$POSTGRES_HOST\"" >> /env.sh +echo "POSTGRES_PORT=$POSTGRES_PORT" >> /env.sh +echo "POSTGRES_ENGINE=\"$POSTGRES_ENGINE\"" >> /env.sh +echo "POSTGRES_POOL_PRE_PING=\"$POSTGRES_POOL_PRE_PING\"" >> /env.sh +echo "POSTGRES_POOL_SIZE=$POSTGRES_POOL_SIZE" >> /env.sh +echo "POSTGRES_MAX_OVERFLOW=$POSTGRES_MAX_OVERFLOW" >> /env.sh +echo "POSTGRES_POOL_RECYCLE=$POSTGRES_POOL_RECYCLE" >> /env.sh +echo "POSTGRES_POOL_TIMEOUT=$POSTGRES_POOL_TIMEOUT" >> /env.sh +echo "POSTGRES_ECHO=\"$POSTGRES_ECHO\"" >> /env.sh + +# Add Python environment variables +echo "PYTHONPATH=/" >> /env.sh +echo "PYTHONUNBUFFERED=1" >> /env.sh +echo "PYTHONDONTWRITEBYTECODE=1" >> /env.sh + +# Make the environment file available to cron +echo "*/15 * * * * /run_app.sh >> /var/log/cron.log 2>&1" > /tmp/crontab_list +crontab /tmp/crontab_list + +# Start cron +cron + +# Tail the log file +tail -f /var/log/cron.log diff --git a/trash/XDecisionBook/run_app.sh b/trash/XDecisionBook/run_app.sh new file mode 100644 index 0000000..83dcf19 --- /dev/null +++ b/trash/XDecisionBook/run_app.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Source the environment file directly +. /env.sh + +# Re-export all variables to ensure they're available to the Python script +export POSTGRES_USER +export POSTGRES_PASSWORD +export POSTGRES_DB +export POSTGRES_HOST +export POSTGRES_PORT +export POSTGRES_ENGINE +export POSTGRES_POOL_PRE_PING +export POSTGRES_POOL_SIZE +export POSTGRES_MAX_OVERFLOW +export POSTGRES_POOL_RECYCLE +export POSTGRES_POOL_TIMEOUT +export POSTGRES_ECHO + +# Python environment variables +export PYTHONPATH +export PYTHONUNBUFFERED +export PYTHONDONTWRITEBYTECODE + +# env >> /var/log/cron.log +/usr/local/bin/python /runner.py diff --git a/ServicesBank/Finder/DecisionBook/runner.py b/trash/XDecisionBook/runner.py similarity index 100% rename from ServicesBank/Finder/DecisionBook/runner.py rename to trash/XDecisionBook/runner.py