evyos-center-server/prod-wag-backend-automate-services
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

black shift

Browse Source
This commit is contained in:
berkay
2025-04-22 11:10:29 +03:00
parent d7f1da8de8
commit e5f88f2eb4
30 changed files with 671 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
BankServices/EmailService/app.py
View File

@@ -30,46 +30,46 @@ T = TypeVar("T")
class EmailProcessingContext:
"""Context manager for email processing that marks emails as unread if an error occurs."""
def __init__(self, email_message, mark_as_read: bool = True):
self.email_message = email_message
self.mark_as_read = mark_as_read
self.success = False
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None or not self.success:
# If an exception occurred or processing wasn't successful, mark as unread
try:
if hasattr(self.email_message, 'mark_as_unread'):
if hasattr(self.email_message, "mark_as_unread"):
self.email_message.mark_as_unread()
print(f"[EMAIL_SERVICE] Marked email as UNREAD due to processing error: {exc_val if exc_val else 'Unknown error'}")
print(
f"[EMAIL_SERVICE] Marked email as UNREAD due to processing error: {exc_val if exc_val else 'Unknown error'}"
)
except Exception as e:
print(f"[EMAIL_SERVICE] Failed to mark email as unread: {str(e)}")
elif self.mark_as_read:
# If processing was successful and mark_as_read is True, ensure it's marked as read
try:
if hasattr(self.email_message, 'mark_as_read'):
if hasattr(self.email_message, "mark_as_read"):
self.email_message.mark_as_read()
except Exception as e:
print(f"[EMAIL_SERVICE] Failed to mark email as read: {str(e)}")
return False # Don't suppress exceptions
def publish_payload_to_redis(
payload, filename: str, mail_info: dict
) -> bool:
def publish_payload_to_redis(payload, filename: str, mail_info: dict) -> bool:
# Create message document
# Use base64 encoding for binary payloads to ensure proper transmission
if isinstance(payload, bytes):
encoded_payload = base64.b64encode(payload).decode('utf-8')
encoded_payload = base64.b64encode(payload).decode("utf-8")
is_base64 = True
else:
encoded_payload = payload
is_base64 = False
message = {
"filename": filename,
"payload": encoded_payload,
@@ -79,12 +79,14 @@ def publish_payload_to_redis(
"uuid": str(uuid4()), # Use UUID
**mail_info,
}
# Publish to Redis channel
result = redis_pubsub.publisher.publish(REDIS_CHANNEL, message)
if result.status:
print(f"[EMAIL_SERVICE] Published message with filename: {filename} to channel: {REDIS_CHANNEL}")
print(
f"[EMAIL_SERVICE] Published message with filename: {filename} to channel: {REDIS_CHANNEL}"
)
return True
else:
print(f"[EMAIL_SERVICE] Publish error: {result.error}")
@@ -126,7 +128,7 @@ def app():
port = Config.EMAIL_PORT
username = Config.EMAIL_USERNAME
password = Config.EMAIL_PASSWORD
box = EmailBox(host=host, port=port, username=username, password=password)
if not box:
return Exception("Mailbox not found")
@@ -136,41 +138,51 @@ def app():
filter_mail = OR(FROM(Config.MAILBOX), FROM(Config.MAIN_MAIL))
filter_print = f"{Config.MAILBOX} & {Config.MAIN_MAIL}"
# Determine if this is the first run of the day
# Determine if this is the first run of the day
# Store last run date in a file
last_run_file = "/tmp/email_service_last_run.json"
current_date = datetime.now().strftime("%Y-%m-%d")
days_to_check, full_check = 7, 90 # Default to 7 days
try:
if os.path.exists(last_run_file):
with open(last_run_file, 'r') as f:
with open(last_run_file, "r") as f:
last_run_data = json.load(f)
last_run_date = last_run_data.get('last_run_date')
last_run_date = last_run_data.get("last_run_date")
# If this is the first run of a new day, check 90 days
if last_run_date != current_date:
days_to_check = full_check
print(f"[EMAIL_SERVICE] First run of the day. Checking emails from the past {days_to_check} days")
print(
f"[EMAIL_SERVICE] First run of the day. Checking emails from the past {days_to_check} days"
)
else:
print(f"[EMAIL_SERVICE] Subsequent run today. Checking emails from the past {days_to_check} days")
print(
f"[EMAIL_SERVICE] Subsequent run today. Checking emails from the past {days_to_check} days"
)
else:
# If no last run file exists, this is the first run ever - check 90 days
days_to_check = full_check
print(f"[EMAIL_SERVICE] First run detected. Checking emails from the past {days_to_check} days")
print(
f"[EMAIL_SERVICE] First run detected. Checking emails from the past {days_to_check} days"
)
except Exception as e:
print(f"[EMAIL_SERVICE] Error reading last run file: {str(e)}. Using default of {days_to_check} days")
print(
f"[EMAIL_SERVICE] Error reading last run file: {str(e)}. Using default of {days_to_check} days"
)
# Update the last run file
try:
with open(last_run_file, 'w') as f:
json.dump({'last_run_date': current_date}, f)
with open(last_run_file, "w") as f:
json.dump({"last_run_date": current_date}, f)
except Exception as e:
print(f"[EMAIL_SERVICE] Error writing last run file: {str(e)}")
# Calculate the date to check from
check_since_date = (datetime.now() - timedelta(days=days_to_check)).strftime("%d-%b-%Y")
check_since_date = (datetime.now() - timedelta(days=days_to_check)).strftime(
"%d-%b-%Y"
)
for folder in mail_folders:
if folder.name == "INBOX":
# Search for emails since the calculated date
@@ -184,27 +196,33 @@ def app():
# Use context manager to handle errors and mark email as unread if needed
with EmailProcessingContext(banks_mail) as ctx:
try:
headers = {k.lower(): v for k, v in banks_mail.headers.items()}
headers = {
k.lower(): v for k, v in banks_mail.headers.items()
}
mail_info = {
"from": headers["from"],
"to": headers["to"],
"subject": headers["subject"],
"date": str(headers["date"]),
}
# Process the email and publish to Redis
success = read_email_and_publish_to_redis(
email_message=email_message, mail_info=mail_info
)
# Set success flag for the context manager
ctx.success = success
if success:
print(f"[EMAIL_SERVICE] Successfully processed email with subject: {mail_info['subject']}")
print(
f"[EMAIL_SERVICE] Successfully processed email with subject: {mail_info['subject']}"
)
else:
print(f"[EMAIL_SERVICE] No matching attachments found in email with subject: {mail_info['subject']}")
print(
f"[EMAIL_SERVICE] No matching attachments found in email with subject: {mail_info['subject']}"
)
except Exception as e:
print(f"[EMAIL_SERVICE] Error processing email: {str(e)}")
# The context manager will mark the email as unread
@@ -213,8 +231,8 @@ def app():
if __name__ == "__main__":
print("=== Starting Email Service with Redis Pub/Sub ===")
print(f"Publishing to channel: {REDIS_CHANNEL}")
time.sleep(20) # Wait for 20 seconds to other services to kick in
time.sleep(20) # Wait for 20 seconds to other services to kick in
while True:
print("\n[EMAIL_SERVICE] Checking for new emails...")
app()

110
BankServices/ParserService/app.py
View File

@@ -18,14 +18,16 @@ REDIS_CHANNEL_OUT = "parser" # Publish to Parser Service channel
delimiter = "|"
def publish_parsed_data_to_redis(data, collected_data_dict: list[dict], filename: str) -> bool:
def publish_parsed_data_to_redis(
data, collected_data_dict: list[dict], filename: str
) -> bool:
"""Publish parsed data to Redis.
Args:
data: Original message data from Redis
collected_data_dict: Parsed data from Excel file
filename: Name of the processed file
Returns:
bool: Success status
"""
@@ -40,16 +42,18 @@ def publish_parsed_data_to_redis(data, collected_data_dict: list[dict], filename
else:
message["parsed"] = None
message["stage"] = "not found" # Mark as 'not found' if parsing failed
# Add processing timestamp
message["parsed_at"] = str(arrow.now())
message["filename"] = filename
# Publish to Redis channel
result = redis_pubsub.publisher.publish(REDIS_CHANNEL_OUT, message)
if result.status:
print(f"[PARSER_SERVICE] Published parsed data for {filename} with stage: {message['stage']}")
print(
f"[PARSER_SERVICE] Published parsed data for {filename} with stage: {message['stage']}"
)
return True
else:
print(f"[PARSER_SERVICE] Publish error: {result.error}")
@@ -58,10 +62,10 @@ def publish_parsed_data_to_redis(data, collected_data_dict: list[dict], filename
def parse_excel_file(excel_frame: DataFrame) -> list[dict]:
"""Parse Excel file data.
Args:
excel_frame: DataFrame containing Excel data
Returns:
list[dict]: List of parsed data dictionaries
"""
@@ -76,13 +80,17 @@ def parse_excel_file(excel_frame: DataFrame) -> list[dict]:
dict(
iban=str(iban),
bank_date=arrow.get(
datetime.datetime.strptime(str(row[1]), "%d/%m/%Y-%H:%M:%S")
datetime.datetime.strptime(
str(row[1]), "%d/%m/%Y-%H:%M:%S"
)
).__str__(),
channel_branch=unidecode(str(row[3])),
currency_value=(
float(str(row[4]).replace(",", "")) if row[4] else 0
),
balance=float(str(row[5]).replace(",", "")) if row[5] else 0,
balance=(
float(str(row[5]).replace(",", "")) if row[5] else 0
),
additional_balance=(
float(str(row[6]).replace(",", "")) if row[6] else 0
),
@@ -92,7 +100,9 @@ def parse_excel_file(excel_frame: DataFrame) -> list[dict]:
bank_reference_code=str(row[15]),
)
)
print(f"[PARSER_SERVICE] Successfully parsed {len(data_list)} records from Excel file")
print(
f"[PARSER_SERVICE] Successfully parsed {len(data_list)} records from Excel file"
)
except Exception as e:
print(f"[PARSER_SERVICE] Error parsing Excel file: {str(e)}")
return data_list
@@ -100,13 +110,13 @@ def parse_excel_file(excel_frame: DataFrame) -> list[dict]:
def process_message(message):
"""Process a message from Redis.
Args:
message: Message data from Redis subscriber
"""
# Extract the message data
data = message["data"]
# If data is a string, parse it as JSON
if isinstance(data, str):
try:
@@ -114,7 +124,7 @@ def process_message(message):
except json.JSONDecodeError as e:
print(f"[PARSER_SERVICE] Error parsing message data: {e}")
return
# Check if stage is 'red' before processing
if data.get("stage") == "red":
try:
@@ -122,91 +132,109 @@ def process_message(message):
payload = data.get("payload")
is_base64 = data.get("is_base64", False)
print(f"[PARSER_SERVICE] Processing file: {filename}")
# Handle base64-encoded payload
if is_base64 and isinstance(payload, str):
try:
# Decode base64 string to bytes
payload = base64.b64decode(payload)
print(f"[PARSER_SERVICE] Successfully decoded base64 payload, size: {len(payload)} bytes")
print(
f"[PARSER_SERVICE] Successfully decoded base64 payload, size: {len(payload)} bytes"
)
except Exception as e:
print(f"[PARSER_SERVICE] Error decoding base64 payload: {str(e)}")
# Convert regular string payload to bytes if needed
elif isinstance(payload, str):
payload = payload.encode('utf-8')
payload = payload.encode("utf-8")
# Create an in-memory file-like object and try multiple approaches
excel_frame = None
errors = []
# Save payload to a temporary file for debugging if needed
temp_file_path = f"/tmp/{filename}"
try:
with open(temp_file_path, 'wb') as f:
with open(temp_file_path, "wb") as f:
f.write(payload)
print(f"[PARSER_SERVICE] Saved payload to {temp_file_path} for debugging")
print(
f"[PARSER_SERVICE] Saved payload to {temp_file_path} for debugging"
)
except Exception as e:
print(f"[PARSER_SERVICE] Could not save debug file: {str(e)}")
# Try different approaches to read the Excel file
approaches = [
# Approach 1: Try xlrd for .xls files
lambda: DataFrame(read_excel(io.BytesIO(payload), engine='xlrd')) if filename.lower().endswith('.xls') else None,
lambda: (
DataFrame(read_excel(io.BytesIO(payload), engine="xlrd"))
if filename.lower().endswith(".xls")
else None
),
# Approach 2: Try openpyxl for .xlsx files
lambda: DataFrame(read_excel(io.BytesIO(payload), engine='openpyxl')) if filename.lower().endswith('.xlsx') else None,
lambda: (
DataFrame(read_excel(io.BytesIO(payload), engine="openpyxl"))
if filename.lower().endswith(".xlsx")
else None
),
# Approach 3: Try xlrd with explicit sheet name
lambda: DataFrame(read_excel(io.BytesIO(payload), engine='xlrd', sheet_name=0)),
lambda: DataFrame(
read_excel(io.BytesIO(payload), engine="xlrd", sheet_name=0)
),
# Approach 4: Try with temporary file
lambda: DataFrame(read_excel(temp_file_path)),
]
# Try each approach until one works
for i, approach in enumerate(approaches):
try:
result = approach()
if result is not None:
excel_frame = result
print(f"[PARSER_SERVICE] Successfully read Excel file using approach {i+1}")
print(
f"[PARSER_SERVICE] Successfully read Excel file using approach {i+1}"
)
break
except Exception as e:
errors.append(f"Approach {i+1}: {str(e)}")
# If all approaches failed, raise an exception
if excel_frame is None:
error_details = "\n".join(errors)
raise Exception(f"Failed to read Excel file using all approaches:\n{error_details}")
raise Exception(
f"Failed to read Excel file using all approaches:\n{error_details}"
)
# Extract data from the Excel file
collected_data_dict = parse_excel_file(excel_frame)
# Publish parsed data to Redis
publish_parsed_data_to_redis(
data=data,
collected_data_dict=collected_data_dict,
filename=filename
data=data, collected_data_dict=collected_data_dict, filename=filename
)
except Exception as e:
print(f"[PARSER_SERVICE] Error processing message: {str(e)}")
else:
print(f"[PARSER_SERVICE] Skipped message with UUID: {data.get('uuid')} (stage is not 'red')")
print(
f"[PARSER_SERVICE] Skipped message with UUID: {data.get('uuid')} (stage is not 'red')"
)
def app():
"""Main application function."""
print("[PARSER_SERVICE] Starting Parser Service")
# Subscribe to the input channel
result = redis_pubsub.subscriber.subscribe(REDIS_CHANNEL_IN, process_message)
if result.status:
print(f"[PARSER_SERVICE] Subscribed to channel: {REDIS_CHANNEL_IN}")
else:
print(f"[PARSER_SERVICE] Subscribe error: {result.error}")
return
# Start listening for messages
listen_result = redis_pubsub.subscriber.start_listening(in_thread=True)
if listen_result.status:
print("[PARSER_SERVICE] Listening for messages")
else:
@@ -217,7 +245,7 @@ def app():
if __name__ == "__main__":
# Initialize the app once
app()
# Keep the main thread alive
try:
while True:

71
BankServices/RoutineEmailService/app.py
View File

@@ -11,13 +11,13 @@ def render_email_template(
) -> str:
"""
Render the HTML email template with the provided data.
Args:
headers: List of column headers for the table
rows: List of data rows for the table
balance_error: Flag indicating if there's a balance discrepancy
bank_balance: Current bank balance formatted as string
Returns:
Rendered HTML template as string
"""
@@ -25,7 +25,7 @@ def render_email_template(
# Look for template in ServiceDepends directory
env = Environment(loader=FileSystemLoader("/"))
template = env.get_template("template_accounts.html")
# Render template with variables
return template.render(
headers=headers,
@@ -35,7 +35,7 @@ def render_email_template(
today=str(arrow.now().date()),
)
except Exception as e:
print('Exception render template:',e)
print("Exception render template:", e)
err = e
raise
@@ -43,25 +43,25 @@ def render_email_template(
def send_email_to_given_address(send_to: str, html_template: str) -> bool:
"""
Send email with the rendered HTML template to the specified address.
Args:
send_to: Email address of the recipient
html_template: Rendered HTML template content
Returns:
Boolean indicating if the email was sent successfully
"""
today = arrow.now()
subject = f"{str(today.date())} Gunes Apt. Cari Durum Bilgilendirme Raporu"
# Create email parameters using EmailSendModel
email_params = EmailSendModel(
subject=subject,
html=html_template,
receivers=[send_to],
text=f"Gunes Apt. Cari Durum Bilgilendirme Raporu - {today.date()}"
text=f"Gunes Apt. Cari Durum Bilgilendirme Raporu - {today.date()}",
)
try:
# Use the context manager to handle connection errors
with EmailService.new_session() as email_session:
@@ -69,63 +69,74 @@ def send_email_to_given_address(send_to: str, html_template: str) -> bool:
EmailService.send_email(email_session, email_params)
return True
except Exception as e:
print(f'Exception send email: {e}')
print(f"Exception send email: {e}")
return False
def set_account_records_to_send_email() -> bool:
"""
Retrieve account records from the database, format them, and send an email report.
Usage:
from app import set_account_records_to_send_email
Returns:
Boolean indicating if the process completed successfully
"""
# Get database session and retrieve records
with AccountRecords.new_session() as db_session:
account_records_query = AccountRecords.filter_all(db=db_session).query
# Get the 3 most recent records
account_records: List[AccountRecords] | [] = (
account_records_query.order_by(
AccountRecords.bank_date.desc(),
AccountRecords.bank_reference_code.desc()
AccountRecords.bank_date.desc(),
AccountRecords.bank_reference_code.desc(),
)
.limit(3)
.all()
)
# Check if we have enough records
if len(account_records) < 2:
return False
# Check for balance discrepancy
first_record, second_record = account_records[0], account_records[1]
expected_second_balance = first_record.bank_balance - first_record.currency_value
expected_second_balance = (
first_record.bank_balance - first_record.currency_value
)
balance_error = expected_second_balance != second_record.bank_balance
if balance_error:
return False
# Format rows for the email template
list_of_rows = []
for record in account_records:
list_of_rows.append([
record.bank_date.strftime("%d/%m/%Y %H:%M"),
record.process_comment,
f"{record.currency_value:,.2f}",
f"{record.bank_balance:,.2f}"
])
list_of_rows.append(
[
record.bank_date.strftime("%d/%m/%Y %H:%M"),
record.process_comment,
f"{record.currency_value:,.2f}",
f"{record.bank_balance:,.2f}",
]
)
# Get the most recent bank balance
last_bank_balance = sorted(account_records, key=lambda x: x.bank_date, reverse=True)[0].bank_balance
last_bank_balance = sorted(
account_records, key=lambda x: x.bank_date, reverse=True
)[0].bank_balance
# Define headers for the table
headers = ["Ulaştığı Tarih", "Banka Transaksiyonu Ek Bilgi", "Aktarım Değeri", "Banka Bakiyesi"]
headers = [
"Ulaştığı Tarih",
"Banka Transaksiyonu Ek Bilgi",
"Aktarım Değeri",
"Banka Bakiyesi",
]
# Recipient email address
send_to = "karatay@mehmetkaratay.com.tr"
# Render email template
html_template = render_email_template(
headers=headers,
@@ -133,7 +144,7 @@ def set_account_records_to_send_email() -> bool:
balance_error=balance_error,
bank_balance=f"{last_bank_balance:,.2f}",
)
# Send the email
return send_email_to_given_address(send_to=send_to, html_template=html_template)

98
BankServices/SenderService/app.py
View File

@@ -12,13 +12,13 @@ def render_email_template(
) -> str:
"""
Render the HTML email template with the provided data.
Args:
headers: List of column headers for the table
rows: List of data rows for the table
balance_error: Flag indicating if there's a balance discrepancy
bank_balance: Current bank balance formatted as string
Returns:
Rendered HTML template as string
"""
@@ -26,7 +26,7 @@ def render_email_template(
# Look for template in ServiceDepends directory
env = Environment(loader=FileSystemLoader("/"))
template = env.get_template("template_accounts.html")
return template.render(
headers=headers,
rows=rows,
@@ -35,32 +35,34 @@ def render_email_template(
today=str(arrow.now().date()),
)
except Exception as e:
print(f'Template rendering failed: {e}')
print(f"Template rendering failed: {e}")
raise
def send_email_to_given_address(send_to: str, html_template: str, count_of_records: int) -> bool:
def send_email_to_given_address(
send_to: str, html_template: str, count_of_records: int
) -> bool:
"""
Send email with the rendered HTML template to the specified address.
Args:
send_to: Email address of the recipient
html_template: Rendered HTML template content
Returns:
Boolean indicating if the email was sent successfully
"""
today = arrow.now()
subject = f"{str(today.date())} Gunes Apt. Cari Durum Kayıt Giriş Raporu"
# Create email parameters using EmailSendModel
email_params = EmailSendModel(
subject=subject + f" ({count_of_records} kayıt)",
html=html_template,
receivers=[send_to],
text=f"Gunes Apt. Cari Durum Kayıt Giriş Raporu - {today.date()}"
text=f"Gunes Apt. Cari Durum Kayıt Giriş Raporu - {today.date()}",
)
try:
# Use the context manager to handle connection errors
with EmailService.new_session() as email_session:
@@ -69,17 +71,17 @@ def send_email_to_given_address(send_to: str, html_template: str, count_of_recor
print(f"Email successfully sent to: {send_to}")
return True
except Exception as e:
print(f'Failed to send email: {e}')
print(f"Failed to send email: {e}")
return False
def process_unsent_email_records() -> bool:
"""
Process account records that haven't been emailed yet.
Finds records with is_email_send=False, formats them into an email,
sends the email, and updates the records as sent if successful.
Returns:
bool: True if email was sent successfully, False otherwise
"""
@@ -87,42 +89,55 @@ def process_unsent_email_records() -> bool:
# Use the context manager to handle database connections
with AccountRecords.new_session() as db_session:
# Query un-sent mail rows - with limit for display only
account_records_query = AccountRecords.filter_all(
AccountRecords.is_email_send == False,
db=db_session,
).query.order_by(AccountRecords.bank_date.asc()).limit(20)
account_records_query = (
AccountRecords.filter_all(
AccountRecords.is_email_send == False,
db=db_session,
)
.query.order_by(AccountRecords.bank_date.asc())
.limit(20)
)
account_records: List[AccountRecords] = account_records_query.all()
if not account_records:
print("No unsent email records found")
return False
# Get the IDs of the records we're processing
record_ids = [record.id for record in account_records]
print(f"Found {len(account_records)} unsent email records")
# Format rows for the email template
list_of_rows = []
for record in account_records:
list_of_rows.append([
record.bank_date.strftime("%d/%m/%Y %H:%M"),
record.process_comment,
f"{record.currency_value:,.2f}",
f"{record.bank_balance:,.2f}"
])
list_of_rows.append(
[
record.bank_date.strftime("%d/%m/%Y %H:%M"),
record.process_comment,
f"{record.currency_value:,.2f}",
f"{record.bank_balance:,.2f}",
]
)
# Reverse list by date
list_of_rows = list_of_rows[::-1]
# Get the most recent bank balance
last_bank_balance = sorted(account_records, key=lambda x: x.bank_date, reverse=True)[0].bank_balance
last_bank_balance = sorted(
account_records, key=lambda x: x.bank_date, reverse=True
)[0].bank_balance
# Define headers for the table
headers = ["Ulaştığı Tarih", "Banka Transaksiyonu Ek Bilgi", "Aktarım Değeri", "Banka Bakiyesi"]
headers = [
"Ulaştığı Tarih",
"Banka Transaksiyonu Ek Bilgi",
"Aktarım Değeri",
"Banka Bakiyesi",
]
# Recipient email address
send_to = "karatay@mehmetkaratay.com.tr"
# Render email template
html_template = render_email_template(
headers=headers,
@@ -130,32 +145,35 @@ def process_unsent_email_records() -> bool:
balance_error=False,
bank_balance=f"{last_bank_balance:,.2f}",
)
# Send the email
if send_email_to_given_address(send_to=send_to, html_template=html_template, count_of_records=len(list_of_rows)):
if send_email_to_given_address(
send_to=send_to,
html_template=html_template,
count_of_records=len(list_of_rows),
):
# Create a new query without limit for updating
update_query = AccountRecords.filter_all(
AccountRecords.id.in_(record_ids),
db=db_session
AccountRecords.id.in_(record_ids), db=db_session
).query
# Update records as sent
update_query.update({"is_email_send": True}, synchronize_session=False)
AccountRecords.save(db_session)
print(f"Successfully marked {len(account_records)} records as sent")
return True
print("Email sending failed, records not updated")
return False
except Exception as e:
print(f'Error processing unsent email records: {e}')
print(f"Error processing unsent email records: {e}")
return False
if __name__ == "__main__":
print("Starting Email Sender Service")
while True:
try:
result = process_unsent_email_records()
@@ -165,7 +183,7 @@ if __name__ == "__main__":
print("No emails sent in this iteration")
except Exception as e:
print(f"Unexpected error in main loop: {e}")
# Sleep for 60 seconds before next check
print("Sleeping for 60 seconds")
time.sleep(60)

2
BankServices/ServiceDepends/config.py
View File

@@ -22,7 +22,7 @@ class Config:
class EmailConfig:
EMAIL_HOST: str = os.getenv("EMAIL_HOST", "10.10.2.34")
EMAIL_HOST: str = os.getenv("EMAIL_HOST", "10.10.2.34")
EMAIL_USERNAME: str = Config.EMAIL_SENDER_USERNAME
EMAIL_PASSWORD: str = Config.EMAIL_PASSWORD
EMAIL_PORT: int = Config.EMAIL_SEND_PORT

72
BankServices/WriterService/app.py
View File

@@ -18,28 +18,30 @@ delimiter = "|"
def publish_written_data_to_redis(data: Dict[str, Any], file_name: str) -> bool:
"""Publish written data status to Redis.
Args:
data: Original message data from Redis
file_name: Name of the processed file
Returns:
bool: Success status
"""
# Create a copy of the original message to preserve metadata
message = data.copy() if isinstance(data, dict) else {}
# Update stage to 'written'
message["stage"] = "written"
# Add processing timestamp
message["written_at"] = str(arrow.now())
# Publish to Redis channel
result = redis_pubsub.publisher.publish(REDIS_CHANNEL_OUT, message)
if result.status:
print(f"[WRITER_SERVICE] Published written status for {file_name} with stage: written")
print(
f"[WRITER_SERVICE] Published written status for {file_name} with stage: written"
)
return True
else:
print(f"[WRITER_SERVICE] Publish error: {result.error}")
@@ -48,10 +50,10 @@ def publish_written_data_to_redis(data: Dict[str, Any], file_name: str) -> bool:
def write_parsed_data_to_account_records(data_dict: dict, file_name: str) -> bool:
"""Write parsed data to account records database.
Args:
data_dict: Parsed data dictionary
Returns:
bool: True if record was created or already exists, False on error
"""
@@ -61,8 +63,8 @@ def write_parsed_data_to_account_records(data_dict: dict, file_name: str) -> boo
data_dict["bank_balance"] = data_dict.pop("balance")
data_dict["import_file_name"] = file_name
data_dict = BankReceive(**data_dict).model_dump()
print('data_dict', data_dict)
print("data_dict", data_dict)
# Process date fields
bank_date = arrow.get(str(data_dict["bank_date"]))
data_dict["bank_date_w"] = bank_date.weekday()
@@ -70,7 +72,7 @@ def write_parsed_data_to_account_records(data_dict: dict, file_name: str) -> boo
data_dict["bank_date_d"] = bank_date.day
data_dict["bank_date_y"] = bank_date.year
data_dict["bank_date"] = str(bank_date)
# Add build information if available
if build_iban := BuildIbans.filter_by_one(
iban=data_dict["iban"], db=db_session
@@ -81,7 +83,7 @@ def write_parsed_data_to_account_records(data_dict: dict, file_name: str) -> boo
"build_uu_id": build_iban.build_uu_id,
}
)
# Create new record or find existing one using specific fields for matching
new_account_record = AccountRecords.find_or_create(
db=db_session,
@@ -90,16 +92,20 @@ def write_parsed_data_to_account_records(data_dict: dict, file_name: str) -> boo
AccountRecords.bank_date,
AccountRecords.iban,
AccountRecords.bank_reference_code,
AccountRecords.bank_balance
]
AccountRecords.bank_balance,
],
)
if new_account_record.meta_data.created:
new_account_record.is_confirmed = True
new_account_record.save(db=db_session)
print(f"[WRITER_SERVICE] Created new record in database: {new_account_record.id}")
print(
f"[WRITER_SERVICE] Created new record in database: {new_account_record.id}"
)
return True
else:
print(f"[WRITER_SERVICE] Record already exists in database: {new_account_record.id}")
print(
f"[WRITER_SERVICE] Record already exists in database: {new_account_record.id}"
)
return False
except Exception as e:
print(f"[WRITER_SERVICE] Error writing to database: {str(e)}")
@@ -108,13 +114,13 @@ def write_parsed_data_to_account_records(data_dict: dict, file_name: str) -> boo
def process_message(message):
"""Process a message from Redis.
Args:
message: Message data from Redis subscriber
"""
# Extract the message data
data = message["data"]
# If data is a string, parse it as JSON
if isinstance(data, str):
try:
@@ -122,51 +128,55 @@ def process_message(message):
except json.JSONDecodeError as e:
print(f"[WRITER_SERVICE] Error parsing message data: {e}")
return
# Check if stage is 'parsed' before processing
if data.get("stage") == "parsed":
try:
file_name = data.get("filename")
parsed_data = data.get("parsed")
print(f"[WRITER_SERVICE] Processing file: {file_name}")
if not parsed_data:
print(f"[WRITER_SERVICE] No parsed data found for {file_name}")
return
# Process each parsed data item
success = True
for item in parsed_data:
result = write_parsed_data_to_account_records(data_dict=item, file_name=file_name)
result = write_parsed_data_to_account_records(
data_dict=item, file_name=file_name
)
if not result:
success = False
# Publish status update to Redis if all records were processed
if success:
publish_written_data_to_redis(data=data, file_name=file_name)
except Exception as e:
print(f"[WRITER_SERVICE] Error processing message: {str(e)}")
else:
print(f"[WRITER_SERVICE] Skipped message with UUID: {data.get('uuid')} (stage is not 'parsed')")
print(
f"[WRITER_SERVICE] Skipped message with UUID: {data.get('uuid')} (stage is not 'parsed')"
)
def app():
"""Main application function."""
print("[WRITER_SERVICE] Starting Writer Service")
# Subscribe to the input channel
result = redis_pubsub.subscriber.subscribe(REDIS_CHANNEL_IN, process_message)
if result.status:
print(f"[WRITER_SERVICE] Subscribed to channel: {REDIS_CHANNEL_IN}")
else:
print(f"[WRITER_SERVICE] Subscribe error: {result.error}")
return
# Start listening for messages
listen_result = redis_pubsub.subscriber.start_listening(in_thread=True)
if listen_result.status:
print("[WRITER_SERVICE] Listening for messages")
else:
@@ -177,7 +187,7 @@ def app():
if __name__ == "__main__":
# Initialize the app once
app()
# Keep the main thread alive
try:
while True: