services api

ServicesBank/Email/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12-slim
+
+WORKDIR /
+
+RUN apt-get update && apt-get install -y --no-install-recommends gcc && rm -rf /var/lib/apt/lists/* && pip install --no-cache-dir poetry
+
+COPY /ServicesBank/Email/pyproject.toml ./pyproject.toml
+
+RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi --no-root --only main && pip cache purge && rm -rf ~/.cache/pypoetry
+
+COPY /ServicesBank/Email /
+COPY /ServicesApi/Controllers /ServicesApi/Controllers
+COPY /ServicesBank/Depends/config.py /ServicesBank/Depends/config.py
+
+ENV PYTHONPATH=/ PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
+
+CMD ["poetry", "run", "python", "app.py"]

ServicesBank/Email/README.md

@@ -0,0 +1,84 @@
+# Email Service
+
+## Overview
+The Email Service is the first component in a Redis pub/sub processing chain for bank-related email automation. It monitors a specified mailbox for emails with attachments, filters them based on IBAN criteria, and publishes the data to a Redis channel for further processing.
+
+## Features
+
+### Email Processing
+- Connects to a configured mailbox using IMAP
+- Implements smart date-based filtering:
+  - Checks emails from the past 14 days on the first run of each day
+  - Checks emails from the past 7 days on subsequent runs within the same day
+- Extracts attachments from emails
+- Filters attachments based on IBAN criteria
+- Uses a context manager to ensure emails are properly handled even during errors
+
+### Redis Integration
+- Publishes messages to a Redis pub/sub channel ("CollectedData")
+- Each message contains:
+  - Unique UUID
+  - Timestamp
+  - Initial stage marker ("red")
+  - Attachment payload and metadata
+- Connects to an external Redis server
+
+### Error Handling
+- Robust error management with context managers
+- Automatic marking of emails as unread if processing fails
+- Comprehensive logging
+
+## Configuration
+
+### Environment Variables
+```
+EMAIL_HOST=10.10.2.34
+EMAIL_USERNAME=isbank@mehmetkaratay.com.tr
+EMAIL_PASSWORD=system
+EMAIL_SLEEP=60
+AUTHORIZE_IBAN=4245-0093333
+REDIS_HOST=10.10.2.15
+REDIS_PORT=6379
+REDIS_PASSWORD=your_strong_password_here
+```
+
+## Deployment
+
+### Docker
+The service is containerized using Docker and can be deployed using the provided Dockerfile and docker-compose configuration.
+
+```bash
+# Build and start the service
+docker compose -f bank-services-docker-compose.yml up -d --build
+
+# View logs
+docker compose -f bank-services-docker-compose.yml logs -f email_service
+
+# Stop the service
+docker compose -f bank-services-docker-compose.yml down
+```
+
+### Service Management
+The `check_bank_services.sh` script provides a simple way to restart the service:
+
+```bash
+./check_bank_services.sh
+```
+
+## Architecture
+
+### Redis Pub/Sub Chain
+This service is the first in a multi-stage processing chain:
+1. **Email Service** (this service): Reads emails, extracts attachments, publishes to Redis with stage="red"
+2. **Processor Service**: Subscribes to stage="red" messages, processes data, republishes with stage="processed"
+3. **Writer Service**: Subscribes to stage="processed" messages, writes data to final destination, marks as stage="completed"
+
+## Development
+
+### Dependencies
+- Python 3.12
+- Redbox (email library)
+- Redis
+
+### State Management
+The service maintains a state file at `/tmp/email_service_last_run.json` to track when it last ran, enabling the smart date-based filtering feature.

ServicesBank/Email/app.py

@@ -0,0 +1,169 @@
+import time
+import arrow
+import os
+import json
+import base64
+
+from uuid import uuid4
+from datetime import datetime, timedelta
+from typing import TypeVar
+from redbox import EmailBox
+from redbox.query import FROM, UNSEEN, OR, SINCE
+
+from ServicesApi.Controllers.Redis.Broadcast.actions import redis_pubsub
+from ServicesBank.Depends.config import Config
+
+
+authorized_iban = Config.AUTHORIZE_IBAN
+authorized_iban_cleaned = authorized_iban.replace("-", "")
+REDIS_CHANNEL = "reader"
+delimiter = "|"
+
+# banks_mails = mailbox.search(from_=filter_mail, unseen=True) bununla denemeyin
+# banks_mails = mailbox.search(FROM(filter_mail) & UNSEEN)
+
+
+T = TypeVar("T")
+
+
+class EmailProcessingContext:
+    """Context manager for email processing that marks emails as unread if an error occurs."""
+
+    def __init__(self, email_message, mark_as_read: bool = True):
+        self.email_message = email_message
+        self.mark_as_read = mark_as_read
+        self.success = False
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None or not self.success:
+            try:
+                if hasattr(self.email_message, "mark_as_unread"):
+                    self.email_message.mark_as_unread()
+                    print(f"[EMAIL_SERVICE] Marked email as UNREAD due to processing error: {exc_val if exc_val else 'Unknown error'}")
+            except Exception as e:
+                print(f"[EMAIL_SERVICE] Failed to mark email as unread: {str(e)}")
+        elif self.mark_as_read:
+            try:
+                if hasattr(self.email_message, "mark_as_read"):
+                    self.email_message.mark_as_read()
+            except Exception as e:
+                print(f"[EMAIL_SERVICE] Failed to mark email as read: {str(e)}")
+        return False
+
+
+def publish_payload_to_redis(payload, filename: str, mail_info: dict) -> bool:
+    if isinstance(payload, bytes):
+        encoded_payload = base64.b64encode(payload).decode("utf-8")
+        is_base64 = True
+    else:
+        encoded_payload = payload
+        is_base64 = False
+
+    message = {
+        "filename": filename, "payload": encoded_payload, "is_base64": is_base64, "stage": "red", 
+        "created_at": str(arrow.now()), "uuid": str(uuid4()), **mail_info,
+    }
+
+    result = redis_pubsub.publisher.publish(REDIS_CHANNEL, message)
+    if result.status:
+        print(f"[EMAIL_SERVICE] Published message with filename: {filename} to channel: {REDIS_CHANNEL}")
+        return True
+    else:
+        print(f"[EMAIL_SERVICE] Publish error: {result.error}")
+        return False
+
+
+def read_email_and_publish_to_redis(email_message, mail_info: dict) -> bool:
+    if email_message.is_multipart():  # Check if email has multipart content
+        for part in email_message.walk():  # Each part can be an attachment
+            content_disposition = part.get("Content-Disposition")
+            if content_disposition and "attachment" in content_disposition:
+                if filename := part.get_filename():
+                    is_iban_in_filename = authorized_iban_cleaned in str(filename)
+                    if is_iban_in_filename:
+                        if payload := part.get_payload(decode=True):
+                            return publish_payload_to_redis(payload=payload, filename=filename, mail_info=mail_info)
+    else:
+        content_disposition = email_message.get("Content-Disposition")
+        if content_disposition and "attachment" in content_disposition:
+            if filename := email_message.get_filename():
+                is_iban_in_filename = authorized_iban_cleaned in str(filename)
+                if is_iban_in_filename:
+                    payload = email_message.get_payload(decode=True)
+                    return publish_payload_to_redis(payload=payload, filename=filename, mail_info=mail_info)
+    return False
+
+
+def app():
+
+    host = Config.EMAIL_HOST
+    port = Config.EMAIL_PORT
+    username = Config.EMAIL_USERNAME
+    password = Config.EMAIL_PASSWORD
+
+    box = EmailBox(host=host, port=port, username=username, password=password)
+    if not box:
+        return Exception("Mailbox not found")
+
+    box.connect()
+    mail_folders = box.mailfolders
+    filter_mail = OR(FROM(Config.MAILBOX), FROM(Config.MAIN_MAIL))
+    filter_print = f"{Config.MAILBOX} & {Config.MAIN_MAIL}"
+    last_run_file = "/tmp/email_service_last_run.json"
+    current_date = datetime.now().strftime("%Y-%m-%d")
+    days_to_check, full_check = 7, 90
+
+    try:
+        if os.path.exists(last_run_file):
+            with open(last_run_file, "r") as f:
+                last_run_data = json.load(f)
+                last_run_date = last_run_data.get("last_run_date")
+                if last_run_date != current_date:
+                    days_to_check = full_check
+                    print(f"[EMAIL_SERVICE] First run of the day. Checking emails from the past {days_to_check} days")
+                else:
+                    print(f"[EMAIL_SERVICE] Subsequent run today. Checking emails from the past {days_to_check} days")
+        else:
+            days_to_check = full_check
+            print(f"[EMAIL_SERVICE] First run detected. Checking emails from the past {days_to_check} days")
+    except Exception as e:
+        print(f"[EMAIL_SERVICE] Error reading last run file: {str(e)}. Using default of {days_to_check} days")
+
+    try:
+        with open(last_run_file, "w") as f:
+            json.dump({"last_run_date": current_date}, f)
+    except Exception as e:
+        print(f"[EMAIL_SERVICE] Error writing last run file: {str(e)}")
+
+    check_since_date = (datetime.now() - timedelta(days=days_to_check)).strftime("%d-%b-%Y")
+    for folder in mail_folders:
+        if folder.name == "INBOX":
+            banks_mails = folder.search(filter_mail & SINCE(check_since_date))
+            print(f"[EMAIL_SERVICE] Reading mailbox [{username}] with mail sender [{filter_print}] since {check_since_date} with count: {len(banks_mails)}")
+            for banks_mail in banks_mails or []:
+                if email_message := banks_mail.email:
+                    with EmailProcessingContext(banks_mail) as ctx:
+                        try:
+                            headers = {k.lower(): v for k, v in banks_mail.headers.items()}
+                            mail_info = {"from": headers["from"], "to": headers["to"], "subject": headers["subject"], "date": str(headers["date"])}
+                            success = read_email_and_publish_to_redis(email_message=email_message, mail_info=mail_info)
+                            ctx.success = success
+                            if success:
+                                print(f"[EMAIL_SERVICE] Successfully processed email with subject: {mail_info['subject']}")
+                            else:
+                                print(f"[EMAIL_SERVICE] No matching attachments found in email with subject: {mail_info['subject']}")
+                        except Exception as e:
+                            print(f"[EMAIL_SERVICE] Error processing email: {str(e)}")
+
+
+if __name__ == "__main__":
+    print("=== Starting Email Service with Redis Pub/Sub ===")
+    print(f"Publishing to channel: {REDIS_CHANNEL}")
+    time.sleep(20)
+    while True:
+        print("\n[EMAIL_SERVICE] Checking for new emails...")
+        app()
+        time.sleep(Config.EMAIL_SLEEP)

ServicesBank/Email/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "emailservice"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "arrow>=1.3.0",
+    "redbox>=0.2.1",
+    "redis>=5.2.1",
+    "pydantic-settings>=2.8.1",
+]