import time import arrow from Schemas import AccountRecords, BuildIbans from Services.MongoService.provider import MongoProvider from model import BankReceive from Configs.mongo import MongoConfig mongo_prefix = "CollectedData" delimiter = "|" def collect_parsed_data_from_mongo_database(mongo_provider) -> list: return mongo_provider.find_many(filter_query={"stage": "parsed"}) def write_parsed_data_to_account_records( file: str, data_dict: dict, collection_name: str, mongo_provider ): db_session = AccountRecords.new_session() data_dict["bank_balance"] = data_dict.pop("balance") data_dict["import_file_name"] = collection_name data_dict = BankReceive(**data_dict).model_dump() bank_date = arrow.get(str(data_dict["bank_date"])) data_dict["bank_date_w"] = bank_date.weekday() data_dict["bank_date_m"] = bank_date.month data_dict["bank_date_d"] = bank_date.day data_dict["bank_date_y"] = bank_date.year data_dict["bank_date"] = str(bank_date) if build_iban := BuildIbans.filter_by_one( iban=data_dict["iban"], db=db_session ).data: data_dict.update( { "build_id": build_iban.build_id, "build_uu_id": build_iban.build_uu_id, } ) if found_record := AccountRecords.filter_one( AccountRecords.bank_date == data_dict["bank_date"], AccountRecords.iban == data_dict["iban"], AccountRecords.bank_reference_code == data_dict["bank_reference_code"], AccountRecords.bank_balance == data_dict["bank_balance"], db=db_session, ).data: print("already @database record", found_record.id) else: new_account_record = AccountRecords.find_or_create(db=db_session, **data_dict) new_account_record.is_confirmed = True new_account_record.save(db=db_session) mongo_provider.update_one( filter_query={"filename": file}, update_data={"$set": {"stage": "written"}}, ) if __name__ == "__main__": print("Writer Service is running") while True: with MongoProvider.mongo_client() as mongo_client: provider = MongoProvider( client=mongo_client, database=MongoConfig.DATABASE_NAME, storage_reason=[mongo_prefix, str(arrow.now().date())], ) results = collect_parsed_data_from_mongo_database(mongo_provider=provider) for result in results: parsed_data = result.get("parsed") file_name = result.get("filename") if not parsed_data: continue write_parsed_data_to_account_records( data_dict=parsed_data, collection_name=provider.collection.name, mongo_provider=provider, file=file_name, ) time.sleep(60)