rabbitmq implemented and tested

ServicesTask/app/services/parser/comment/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+
+WORKDIR /
+
+COPY app/services/parser/comment/pyproject.toml  ./
+COPY app/services/parser/comment/README.md ./
+
+COPY app/core ./app/core
+COPY app/services/common/ ./app/services/common/
+COPY app/services/parser/comment/ ./app/services/parser/comment/
+
+RUN pip install --upgrade pip && pip install --no-cache-dir .
+RUN mkdir -p /app/data
+
+CMD ["python", "-m", "app.services.parser.comment.main"]

ServicesTask/app/services/parser/comment/main.py

@@ -0,0 +1,30 @@
+import asyncio
+
+from app.services.common.service_base_async import ServiceBaseAsync
+
+
+PROCESS_SEC = 10
+
+
+async def handle_mail_publish(svc: ServiceBaseAsync, job: dict):
+    await asyncio.sleep(PROCESS_SEC)
+    print("Parser Mail Consumer parsed:", job)
+    # await svc.ack_current()
+    # await svc.enqueue({"source": "parser-mail", "from_task": job}, "parser-mail-done", routing_key="parser.comment.publish")
+
+
+async def consume_default(svc: ServiceBaseAsync, job):
+    print("Parser Mail Consumer default:", job)
+    await asyncio.sleep(PROCESS_SEC)
+    await svc.ack_current()
+
+
+async def produce(_svc: ServiceBaseAsync):
+    print("Parser Mail Producer produce")
+    await asyncio.sleep(PROCESS_SEC)
+
+
+if __name__ == "__main__":
+
+    svc = ServiceBaseAsync(produce_fn=produce, consume_fn=consume_default, handlers={"mail.service.publish": handle_mail_publish})
+    asyncio.run(svc.run())

ServicesTask/app/services/parser/comment/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dual-queue-services"
+version = "0.1.0"
+description = "Async dual queue system with Redis Streams and SQLite persistence"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    { name = "Berkay Karatay", email = "karatay.berkay@gmail.com" }
+]
+
+dependencies = [
+  "aio-pika>=9.4.1",
+  "prometheus-client>=0.20.0",
+  "uvloop>=0.19.0"
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4",
+    "black>=23.0",
+    "isort>=5.12"
+]
+
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+
+[tool.isort]
+profile = "black"
+
+[tool.setuptools.packages.find]
+where = ["app"]
+include = ["app*"]

ServicesTask/app/services/parser/excel/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+
+WORKDIR /
+
+COPY app/services/parser/excel/pyproject.toml  ./
+COPY app/services/parser/excel/README.md ./
+
+COPY app/core ./app/core
+COPY app/services/common/ ./app/services/common/
+COPY app/services/parser/excel/ ./app/services/parser/excel/
+
+RUN pip install --upgrade pip && pip install --no-cache-dir .
+RUN mkdir -p /app/data
+
+CMD ["python", "-m", "app.services.parser.excel.main"]

ServicesTask/app/services/parser/excel/main.py

@@ -0,0 +1,40 @@
+import os
+import uuid
+import asyncio
+
+from app.services.common.service_base_async import ServiceBaseAsync
+
+
+PRODUCE_BURST = int(os.getenv("PRODUCE_BURST", "10"))
+PRODUCE_ONCE  = os.getenv("PRODUCE_ONCE", "true").lower() == "true"
+EVENT_TYPE    = os.getenv("EVENT_TYPE", "db-mongo")
+PROCESS_SEC = 10
+
+
+async def produce(svc: ServiceBaseAsync):
+    await asyncio.sleep(PROCESS_SEC)
+    print(f"Parser Excel Producer produced {len([1,2])} events to '{svc.produce_key}'")
+
+
+async def handle_from_parser(svc: ServiceBaseAsync, job):
+    print("Parser Excel Consumer from parser:", job)
+    await svc.ack_current()
+    return
+
+
+async def handle_from_mail(svc: ServiceBaseAsync, job):
+    print("Parser Excel Consumer from mail:", job)
+    await svc.ack_current()
+    return
+
+
+async def consume_default(svc, job):
+    print("Parser Excel Consumer default:", job)
+    await svc.ack_current()
+    return
+
+
+if __name__ == "__main__":
+
+    svc = ServiceBaseAsync(produce_fn=produce, consume_fn=consume_default, handlers={"parser.publish": handle_from_parser, "mail.publish": handle_from_mail})
+    asyncio.run(svc.run())

ServicesTask/app/services/parser/excel/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dual-queue-services"
+version = "0.1.0"
+description = "Async dual queue system with Redis Streams and SQLite persistence"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    { name = "Berkay Karatay", email = "karatay.berkay@gmail.com" }
+]
+
+dependencies = [
+  "aio-pika>=9.4.1",
+  "prometheus-client>=0.20.0",
+  "uvloop>=0.19.0"
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4",
+    "black>=23.0",
+    "isort>=5.12"
+]
+
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+
+[tool.isort]
+profile = "black"
+
+[tool.setuptools.packages.find]
+where = ["app"]
+include = ["app*"]

ServicesTask/app/services/parser/mail/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+
+WORKDIR /
+
+COPY app/services/parser/mail/pyproject.toml  ./
+COPY app/services/parser/mail/README.md ./
+
+COPY app/core ./app/core
+COPY app/services/common/ ./app/services/common/
+COPY app/services/parser/mail/ ./app/services/parser/mail/
+
+RUN pip install --upgrade pip && pip install --no-cache-dir .
+RUN mkdir -p /app/data
+
+CMD ["python", "-m", "app.services.parser.mail.main"]

ServicesTask/app/services/parser/mail/main.py

@@ -0,0 +1,42 @@
+import os
+import asyncio
+
+from app.services.common.service_base_async import ServiceBaseAsync
+
+
+PRODUCE_BURST = int(os.getenv("PRODUCE_BURST", "10"))
+PRODUCE_ONCE  = os.getenv("PRODUCE_ONCE", "true").lower() == "true"
+EVENT_TYPE    = os.getenv("EVENT_TYPE", "db-mongo")
+PROCESS_SEC = 10
+
+
+async def produce(svc: ServiceBaseAsync):
+    await asyncio.sleep(PROCESS_SEC)
+    print(f"Parser Mail Producer produced {len([1,2])} events to '{svc.produce_key}'")
+
+
+async def handle_db_publish(svc: ServiceBaseAsync, job):
+    await asyncio.sleep(PROCESS_SEC)
+    await svc.ack_current()
+    print("Parser Mail Consumer from db:", job)
+
+
+async def handle_mongo_publish(svc: ServiceBaseAsync, job):
+    await asyncio.sleep(PROCESS_SEC)
+    await svc.ack_current()
+    print("Parser Mail Consumer from mongo:", job)
+
+
+async def consume_default(svc: ServiceBaseAsync, job):
+    await asyncio.sleep(PROCESS_SEC)
+    print("Parser Mail Consumer default:", job)
+    return
+
+
+if __name__ == "__main__":
+
+    svc = ServiceBaseAsync(
+        produce_fn=produce, consume_fn=consume_default,
+        handlers={"database.service.publish": handle_db_publish, "mongo.service.publish": handle_mongo_publish},
+    )
+    asyncio.run(svc.run())

ServicesTask/app/services/parser/mail/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dual-queue-services"
+version = "0.1.0"
+description = "Async dual queue system with Redis Streams and SQLite persistence"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    { name = "Berkay Karatay", email = "karatay.berkay@gmail.com" }
+]
+
+dependencies = [
+  "aio-pika>=9.4.1",
+  "prometheus-client>=0.20.0",
+  "uvloop>=0.19.0"
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4",
+    "black>=23.0",
+    "isort>=5.12"
+]
+
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+
+[tool.isort]
+profile = "black"
+
+[tool.setuptools.packages.find]
+where = ["app"]
+include = ["app*"]

ServicesTask/app/services/parser/payment/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/app
+
+WORKDIR /
+
+COPY app/services/parser/payment/pyproject.toml  ./
+COPY app/services/parser/payment/README.md ./
+
+COPY app/core ./app/core
+COPY app/services/common/ ./app/services/common/
+COPY app/services/parser/payment/ ./app/services/parser/payment/
+
+RUN pip install --upgrade pip && pip install --no-cache-dir .
+RUN mkdir -p /app/data
+
+CMD ["python", "-m", "app.services.parser.payment.main"]

ServicesTask/app/services/parser/payment/main.py

@@ -0,0 +1,43 @@
+import os
+import asyncio
+
+from app.services.common.service_base_async import ServiceBaseAsync
+
+
+PRODUCE_BURST = int(os.getenv("PRODUCE_BURST", "10"))
+PRODUCE_ONCE  = os.getenv("PRODUCE_ONCE", "true").lower() == "true"
+EVENT_TYPE    = os.getenv("EVENT_TYPE", "db-mongo")
+
+PROCESS_SEC = 10
+
+
+async def produce(svc: ServiceBaseAsync):
+    await asyncio.sleep(PROCESS_SEC)
+    print(f"Parser Payment Producer produced {len([1,2])} events to '{svc.produce_key}'")
+
+
+async def handle_from_parser(svc: ServiceBaseAsync, job):
+    await asyncio.sleep(PROCESS_SEC)
+    print("Parser Payment Consumer from parser:", job)
+    await svc.ack_current()
+    return
+
+
+async def handle_from_mail(svc: ServiceBaseAsync, job):
+    await asyncio.sleep(PROCESS_SEC)
+    print("Parser Payment Consumer from mail:", job)
+    await svc.ack_current()
+    return
+
+
+async def consume_default(svc: ServiceBaseAsync, job):
+    await asyncio.sleep(PROCESS_SEC)
+    print("Parser Payment Consumer default:", job)
+    await svc.ack_current()
+    return
+
+
+if __name__ == "__main__":
+
+    svc = ServiceBaseAsync(produce_fn=produce, consume_fn=consume_default, handlers={"parser.publish": handle_from_parser, "mail.publish": handle_from_mail})
+    asyncio.run(svc.run())

ServicesTask/app/services/parser/payment/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dual-queue-services"
+version = "0.1.0"
+description = "Async dual queue system with Redis Streams and SQLite persistence"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    { name = "Berkay Karatay", email = "karatay.berkay@gmail.com" }
+]
+
+dependencies = [
+  "aio-pika>=9.4.1",
+  "prometheus-client>=0.20.0",
+  "uvloop>=0.19.0"
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4",
+    "black>=23.0",
+    "isort>=5.12"
+]
+
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+
+[tool.isort]
+profile = "black"
+
+[tool.setuptools.packages.find]
+where = ["app"]
+include = ["app*"]