From ed803a2ca5ccb741a5f7fa950ffb81710f0cc28f Mon Sep 17 00:00:00 2001 From: Patryk Hegenberg Date: Wed, 18 Feb 2026 12:47:22 +0100 Subject: [PATCH] refactor: seperate system into single Dockerfiles --- apps/dashboard/dashboard/__init__.py | 0 .../dashboard/dashboard}/app.py | 9 +- apps/dashboard/pyproject.toml | 15 +++ apps/pipeline-api/pipeline_api/__init__.py | 0 .../pipeline-api/pipeline_api/api/__init__.py | 0 .../pipeline-api/pipeline_api/api}/main.py | 12 +-- .../pipeline-api/pipeline_api/main.py | 8 +- apps/pipeline-api/pyproject.toml | 20 ++++ docker-compose.yaml | 21 ++-- docker/dashboard.Dockerfile | 42 ++++++++ docker/pipeline-api.Dockerfile | 41 ++++++++ packages/common/common/__init__.py | 0 .../common/common/collectors}/__init__.py | 0 .../common/common/collectors}/smard.py | 4 +- .../common/common/collectors}/weather.py | 5 +- .../common/common/config}/config.yaml | 0 .../common/common/transformators}/__init__.py | 0 .../common/transformators}/transformator.py | 0 .../common/common/utils}/__init__.py | 0 .../common/common/utils}/config_loader.py | 11 ++- .../common/common/utils}/database.py | 2 +- .../common/common/utils}/request_utils.py | 0 packages/common/pyproject.toml | 15 +++ pyproject.toml | 19 +--- scripts/scheduler.sh | 2 +- uv.lock | 97 +++++++++++++------ 26 files changed, 238 insertions(+), 85 deletions(-) create mode 100644 apps/dashboard/dashboard/__init__.py rename {dashboard => apps/dashboard/dashboard}/app.py (91%) create mode 100644 apps/dashboard/pyproject.toml create mode 100644 apps/pipeline-api/pipeline_api/__init__.py create mode 100644 apps/pipeline-api/pipeline_api/api/__init__.py rename {api => apps/pipeline-api/pipeline_api/api}/main.py (83%) rename main.py => apps/pipeline-api/pipeline_api/main.py (95%) create mode 100644 apps/pipeline-api/pyproject.toml create mode 100644 docker/dashboard.Dockerfile create mode 100644 docker/pipeline-api.Dockerfile create mode 100644 packages/common/common/__init__.py rename {collectors => packages/common/common/collectors}/__init__.py (100%) rename {collectors => packages/common/common/collectors}/smard.py (95%) rename {collectors => packages/common/common/collectors}/weather.py (95%) rename {config => packages/common/common/config}/config.yaml (100%) rename {transformators => packages/common/common/transformators}/__init__.py (100%) rename {transformators => packages/common/common/transformators}/transformator.py (100%) rename {utils => packages/common/common/utils}/__init__.py (100%) rename {utils => packages/common/common/utils}/config_loader.py (76%) rename {utils => packages/common/common/utils}/database.py (98%) rename {utils => packages/common/common/utils}/request_utils.py (100%) create mode 100644 packages/common/pyproject.toml diff --git a/apps/dashboard/dashboard/__init__.py b/apps/dashboard/dashboard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dashboard/app.py b/apps/dashboard/dashboard/app.py similarity index 91% rename from dashboard/app.py rename to apps/dashboard/dashboard/app.py index cea3ecf..2ca3088 100644 --- a/dashboard/app.py +++ b/apps/dashboard/dashboard/app.py @@ -1,15 +1,8 @@ -import sys -from pathlib import Path - import duckdb import polars as pl import streamlit as st -project_root = str(Path(__file__).parent.parent) -if project_root not in sys.path: - sys.path.append(project_root) - -from utils.config_loader import settings +from common.utils.config_loader import settings st.set_page_config(page_title="Strompreis & Netz Dashboard", layout="wide") diff --git a/apps/dashboard/pyproject.toml b/apps/dashboard/pyproject.toml new file mode 100644 index 0000000..e220f15 --- /dev/null +++ b/apps/dashboard/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "dashboard" +version = "0.1.0" +description = "Streamlit dashboard for electricity price data" +dependencies = [ + "streamlit>=1.54.0", + "common", +] + +[tool.uv.sources] +common = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/apps/pipeline-api/pipeline_api/__init__.py b/apps/pipeline-api/pipeline_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/pipeline-api/pipeline_api/api/__init__.py b/apps/pipeline-api/pipeline_api/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/main.py b/apps/pipeline-api/pipeline_api/api/main.py similarity index 83% rename from api/main.py rename to apps/pipeline-api/pipeline_api/api/main.py index 53cc9a9..25cca96 100644 --- a/api/main.py +++ b/apps/pipeline-api/pipeline_api/api/main.py @@ -2,17 +2,9 @@ REST API for accessing processed electricity price and network data. """ -import sys -from pathlib import Path from fastapi import FastAPI, HTTPException - -# Add project root to sys.path -project_root = str(Path(__file__).parent.parent) -if project_root not in sys.path: - sys.path.append(project_root) - -from utils import database as db -from utils.config_loader import settings +from common.utils import database as db +from common.utils.config_loader import settings app = FastAPI( title="Strompreis API", diff --git a/main.py b/apps/pipeline-api/pipeline_api/main.py similarity index 95% rename from main.py rename to apps/pipeline-api/pipeline_api/main.py index 7471c4a..9a100ca 100644 --- a/main.py +++ b/apps/pipeline-api/pipeline_api/main.py @@ -7,10 +7,10 @@ import sys import logging import click import polars as pl -from collectors import smard, weather -from transformators import transformator -from utils import database as db -from utils.config_loader import settings +from common.collectors import smard, weather +from common.transformators import transformator +from common.utils import database as db +from common.utils.config_loader import settings # Structured logging configuration logging.basicConfig( diff --git a/apps/pipeline-api/pyproject.toml b/apps/pipeline-api/pyproject.toml new file mode 100644 index 0000000..646d785 --- /dev/null +++ b/apps/pipeline-api/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "pipeline_api" +version = "0.1.0" +description = "ETL Pipeline and FastAPI for electricity price data" +dependencies = [ + "click>=8.3.1", + "fastapi>=0.128.7", + "pyarrow>=23.0.0", + "requests>=2.32.5", + "tenacity>=9.1.4", + "uvicorn>=0.40.0", + "common", +] + +[tool.uv.sources] +common = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/docker-compose.yaml b/docker-compose.yaml index 3a4a364..bd2c1c6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,36 +1,35 @@ services: pipeline: - build: . + build: + context: . + dockerfile: docker/pipeline-api.Dockerfile user: "1000:1000" volumes: - ./output:/app/output:z - - ./config:/app/config:z command: ["/bin/bash", "scripts/scheduler.sh"] environment: - INTERVAL=3600 restart: unless-stopped api: - build: . + build: + context: . + dockerfile: docker/pipeline-api.Dockerfile user: "1000:1000" ports: - "8000:8000" volumes: - ./output:/app/output:z - - ./config:/app/config:z - command: ["python", "-m", "uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] + command: ["/uvbin/uv", "run", "--frozen", "--no-sync", "--package", "pipeline_api", "python", "-m", "uvicorn", "pipeline_api.api.main:app", "--host", "0.0.0.0", "--port", "8000"] restart: unless-stopped dashboard: - build: . + build: + context: . + dockerfile: docker/dashboard.Dockerfile user: "1000:1000" ports: - "8501:8501" volumes: - ./output:/app/output:z - - ./config:/app/config:z - environment: - - STREAMLIT_BROWSER_GATHER_USAGE_STATS=false - - STREAMLIT_USAGE_STATS_ENABLED=false - command: ["streamlit", "run", "dashboard/app.py", "--server.port", "8501", "--server.address", "0.0.0.0"] restart: unless-stopped diff --git a/docker/dashboard.Dockerfile b/docker/dashboard.Dockerfile new file mode 100644 index 0000000..ab54b8d --- /dev/null +++ b/docker/dashboard.Dockerfile @@ -0,0 +1,42 @@ +# Stage 1: Builder +FROM python:3.11-slim-bookworm AS builder +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvbin/uv + +WORKDIR /app +ENV UV_CACHE_DIR=/app/.cache/uv + +COPY pyproject.toml uv.lock ./ +COPY packages/common ./packages/common +COPY apps/dashboard ./apps/dashboard + +# Create a VALID dummy for the other workspace member for validation +RUN mkdir -p apps/pipeline-api/pipeline_api && \ + echo '[project]\nname = "pipeline_api"\nversion = "0.1.0"\n[build-system]\nrequires = ["hatchling"]\nbuild-backend = "hatchling.build"' > apps/pipeline-api/pyproject.toml + +# Install dependencies into the virtualenv +RUN /uvbin/uv sync --frozen --no-dev --package dashboard + +# Stage 2: Final +FROM python:3.11-slim-bookworm +WORKDIR /app + +# Copy project configuration for uv run +COPY pyproject.toml uv.lock ./ + +# Copy only necessary parts from builder +COPY --from=builder /app/.venv /app/.venv +COPY --from=builder /app/packages/common /app/packages/common +COPY --from=builder /app/apps/dashboard /app/apps/dashboard +COPY --from=builder /uvbin/uv /usr/local/bin/uv + +# Create output directory with proper permissions +RUN mkdir -p output && chmod -R 777 output + +EXPOSE 8501 + +ENV STREAMLIT_BROWSER_GATHER_USAGE_STATS=false +ENV STREAMLIT_USAGE_STATS_ENABLED=false +ENV PATH="/app/.venv/bin:$PATH" + +# --no-sync is required as we don't have the uv cache in the final image +CMD ["uv", "run", "--frozen", "--no-sync", "--package", "dashboard", "streamlit", "run", "apps/dashboard/dashboard/app.py", "--server.port", "8501", "--server.address", "0.0.0.0"] diff --git a/docker/pipeline-api.Dockerfile b/docker/pipeline-api.Dockerfile new file mode 100644 index 0000000..c52cfe6 --- /dev/null +++ b/docker/pipeline-api.Dockerfile @@ -0,0 +1,41 @@ +# Stage 1: Builder +FROM python:3.11-slim-bookworm AS builder +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvbin/uv + +WORKDIR /app +ENV UV_CACHE_DIR=/app/.cache/uv + +COPY pyproject.toml uv.lock ./ +COPY packages/common ./packages/common +COPY apps/pipeline-api ./apps/pipeline-api + +# Create a VALID dummy for the other workspace member for validation +RUN mkdir -p apps/dashboard/dashboard && \ + echo '[project]\nname = "dashboard"\nversion = "0.1.0"\n[build-system]\nrequires = ["hatchling"]\nbuild-backend = "hatchling.build"' > apps/dashboard/pyproject.toml + +# Install dependencies into the virtualenv +RUN /uvbin/uv sync --frozen --no-dev --package pipeline_api + +# Stage 2: Final +FROM python:3.11-slim-bookworm +WORKDIR /app + +# Copy project configuration for uv run +COPY pyproject.toml uv.lock ./ + +# Copy only necessary parts from builder +COPY --from=builder /app/.venv /app/.venv +COPY --from=builder /app/packages/common /app/packages/common +COPY --from=builder /app/apps/pipeline-api /app/apps/pipeline-api +COPY --from=builder /uvbin/uv /usr/local/bin/uv +COPY scripts ./scripts + +# Create output directory with proper permissions +RUN mkdir -p output && chmod -R 777 output + +ENV PYTHONPATH=/app/apps/pipeline-api +ENV INTERVAL=3600 +ENV PATH="/app/.venv/bin:$PATH" + +# --no-sync is required as we don't have the uv cache in the final image +CMD ["uv", "run", "--frozen", "--no-sync", "--package", "pipeline_api", "python", "-m", "pipeline_api.main", "run"] diff --git a/packages/common/common/__init__.py b/packages/common/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/collectors/__init__.py b/packages/common/common/collectors/__init__.py similarity index 100% rename from collectors/__init__.py rename to packages/common/common/collectors/__init__.py diff --git a/collectors/smard.py b/packages/common/common/collectors/smard.py similarity index 95% rename from collectors/smard.py rename to packages/common/common/collectors/smard.py index d095c8c..8cb2678 100644 --- a/collectors/smard.py +++ b/packages/common/common/collectors/smard.py @@ -5,8 +5,8 @@ Collector for SMARD (Electricity Market Data) API. import time import logging import polars as pl -from utils import request_utils -from utils.config_loader import settings +from ..utils import request_utils +from ..utils.config_loader import settings logger = logging.getLogger(__name__) diff --git a/collectors/weather.py b/packages/common/common/collectors/weather.py similarity index 95% rename from collectors/weather.py rename to packages/common/common/collectors/weather.py index 7f65bbd..424493b 100644 --- a/collectors/weather.py +++ b/packages/common/common/collectors/weather.py @@ -6,9 +6,8 @@ import logging from datetime import datetime, timedelta, timezone import polars as pl - -from utils import request_utils -from utils.config_loader import settings +from ..utils import request_utils +from ..utils.config_loader import settings logger = logging.getLogger(__name__) diff --git a/config/config.yaml b/packages/common/common/config/config.yaml similarity index 100% rename from config/config.yaml rename to packages/common/common/config/config.yaml diff --git a/transformators/__init__.py b/packages/common/common/transformators/__init__.py similarity index 100% rename from transformators/__init__.py rename to packages/common/common/transformators/__init__.py diff --git a/transformators/transformator.py b/packages/common/common/transformators/transformator.py similarity index 100% rename from transformators/transformator.py rename to packages/common/common/transformators/transformator.py diff --git a/utils/__init__.py b/packages/common/common/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to packages/common/common/utils/__init__.py diff --git a/utils/config_loader.py b/packages/common/common/utils/config_loader.py similarity index 76% rename from utils/config_loader.py rename to packages/common/common/utils/config_loader.py index 9b13a80..69e1e63 100644 --- a/utils/config_loader.py +++ b/packages/common/common/utils/config_loader.py @@ -39,8 +39,15 @@ class Settings(BaseSettings): database: DatabaseConfig = DatabaseConfig() -def load_config(config_path: str = "config/config.yaml") -> Settings: - path = Path(config_path) +def load_config(config_path: str | None = None) -> Settings: + if config_path: + path = Path(config_path) + else: + # Try local first (dev) then package relative + local_path = Path("config/config.yaml") + pkg_path = Path(__file__).parent.parent / "config" / "config.yaml" + path = local_path if local_path.exists() else pkg_path + if not path.exists(): return Settings() diff --git a/utils/database.py b/packages/common/common/utils/database.py similarity index 98% rename from utils/database.py rename to packages/common/common/utils/database.py index bf76a88..c7d281a 100644 --- a/utils/database.py +++ b/packages/common/common/utils/database.py @@ -5,7 +5,7 @@ DuckDB database interface for Bronze (Raw) and Gold (Combined) layers. import duckdb import polars as pl from contextlib import contextmanager -from utils.config_loader import settings +from .config_loader import settings @contextmanager diff --git a/utils/request_utils.py b/packages/common/common/utils/request_utils.py similarity index 100% rename from utils/request_utils.py rename to packages/common/common/utils/request_utils.py diff --git a/packages/common/pyproject.toml b/packages/common/pyproject.toml new file mode 100644 index 0000000..ad3ba56 --- /dev/null +++ b/packages/common/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "common" +version = "0.1.0" +description = "Shared logic and utilities for the Strompreis Pipeline" +dependencies = [ + "duckdb>=1.4.4", + "polars>=1.38.1", + "pydantic>=2.12.5", + "pydantic-settings>=2.12.0", + "pyyaml>=6.0.3", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/pyproject.toml b/pyproject.toml index 4286830..44b0e65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,12 @@ [project] name = "strompreis-pipline" version = "0.1.0" -description = "Add your description here" +description = "Strompreis Pipeline Workspace" readme = "README.md" requires-python = ">=3.11" -dependencies = [ - "click>=8.3.1", - "duckdb>=1.4.4", - "fastapi>=0.128.7", - "polars>=1.38.1", - "pyarrow>=23.0.0", - "pydantic>=2.12.5", - "pydantic-settings>=2.12.0", - "pyyaml>=6.0.3", - "requests>=2.32.5", - "streamlit>=1.54.0", - "tenacity>=9.1.4", - "uvicorn>=0.40.0", -] + +[tool.uv.workspace] +members = ["packages/*", "apps/*"] [tool.ruff] line-length = 88 diff --git a/scripts/scheduler.sh b/scripts/scheduler.sh index 7ca0848..e2de78c 100644 --- a/scripts/scheduler.sh +++ b/scripts/scheduler.sh @@ -2,7 +2,7 @@ echo "Starte Pipeline Scheduler (Intervall: $INTERVAL Sekunden)" while true; do echo "Führe Pipeline aus: $(date)" - python main.py run + /app/.venv/bin/python -m pipeline_api.main run echo "Pipeline beendet. Warte $INTERVAL Sekunden..." sleep ${INTERVAL:-3600} done diff --git a/uv.lock b/uv.lock index 5192140..3f05b63 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,14 @@ resolution-markers = [ "python_full_version < '3.12'", ] +[manifest] +members = [ + "common", + "dashboard", + "pipeline-api", + "strompreis-pipline", +] + [[package]] name = "altair" version = "6.0.0" @@ -183,6 +191,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "common" +version = "0.1.0" +source = { editable = "packages/common" } +dependencies = [ + { name = "duckdb" }, + { name = "polars" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "duckdb", specifier = ">=1.4.4" }, + { name = "polars", specifier = ">=1.38.1" }, + { name = "pydantic", specifier = ">=2.12.5" }, + { name = "pydantic-settings", specifier = ">=2.12.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, +] + +[[package]] +name = "dashboard" +version = "0.1.0" +source = { editable = "apps/dashboard" } +dependencies = [ + { name = "common" }, + { name = "streamlit" }, +] + +[package.metadata] +requires-dist = [ + { name = "common", editable = "packages/common" }, + { name = "streamlit", specifier = ">=1.54.0" }, +] + [[package]] name = "duckdb" version = "1.4.4" @@ -665,6 +709,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/26/c56ce33ca856e358d27fda9676c055395abddb82c35ac0f593877ed4562e/pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", size = 7029880, upload-time = "2026-02-11T04:23:04.783Z" }, ] +[[package]] +name = "pipeline-api" +version = "0.1.0" +source = { editable = "apps/pipeline-api" } +dependencies = [ + { name = "click" }, + { name = "common" }, + { name = "fastapi" }, + { name = "pyarrow" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.3.1" }, + { name = "common", editable = "packages/common" }, + { name = "fastapi", specifier = ">=0.128.7" }, + { name = "pyarrow", specifier = ">=23.0.0" }, + { name = "requests", specifier = ">=2.32.5" }, + { name = "tenacity", specifier = ">=9.1.4" }, + { name = "uvicorn", specifier = ">=0.40.0" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1266,20 +1335,6 @@ wheels = [ name = "strompreis-pipline" version = "0.1.0" source = { virtual = "." } -dependencies = [ - { name = "click" }, - { name = "duckdb" }, - { name = "fastapi" }, - { name = "polars" }, - { name = "pyarrow" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "streamlit" }, - { name = "tenacity" }, - { name = "uvicorn" }, -] [package.dev-dependencies] dev = [ @@ -1291,20 +1346,6 @@ dev = [ ] [package.metadata] -requires-dist = [ - { name = "click", specifier = ">=8.3.1" }, - { name = "duckdb", specifier = ">=1.4.4" }, - { name = "fastapi", specifier = ">=0.128.7" }, - { name = "polars", specifier = ">=1.38.1" }, - { name = "pyarrow", specifier = ">=23.0.0" }, - { name = "pydantic", specifier = ">=2.12.5" }, - { name = "pydantic-settings", specifier = ">=2.12.0" }, - { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "requests", specifier = ">=2.32.5" }, - { name = "streamlit", specifier = ">=1.54.0" }, - { name = "tenacity", specifier = ">=9.1.4" }, - { name = "uvicorn", specifier = ">=0.40.0" }, -] [package.metadata.requires-dev] dev = [