chore: PRWLR-4055 Update project structure and add env vars (#12)

* chore: PRWLR-4055 update project structure and add env vars

* chore: PRWLR-4055 make production settings default

* chore: PRWLR-4055 change default values to env variables

* chore: PRWLR-4055 apply requested changes on default values

* chore: PRWLR-4055 adapt testing environment

.env.example

@@ -0,0 +1,15 @@
+# Django settings
+DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1
+DJANGO_BIND_ADDRESS=0.0.0.0
+DJANGO_PORT=8000
+DJANGO_DEBUG=False
+DJANGO_SETTINGS_MODULE=config.django.[production|devel]
+DJANGO_LOGGING_FORMATTER=[ndjson|human_readable]
+DJANGO_WORKERS=4 # Defaults to the maximum available based on CPU cores if not set.
+
+# PostgreSQL settings
+POSTGRES_HOST=postgres-db
+POSTGRES_PORT=5432
+POSTGRES_USER=prowler
+POSTGRES_PASSWORD=S3cret
+POSTGRES_DB=prowler_db

docker-compose.yml

@@ -3,8 +3,11 @@ services:
     build:
       dockerfile: Dockerfile
     image: prowler-restful-api
+    env_file:
+      - path: ./.env
+        required: false
     ports:
-      - "8000:8000"
+      - "${DJANGO_PORT:-8000}:${DJANGO_PORT:-8000}"
     profiles:
       - prod
     depends_on:
@@ -19,8 +22,14 @@ services:
       dockerfile: Dockerfile
       target: dev
     image: prowler-restful-api-dev
+    environment:
+      - DJANGO_SETTINGS_MODULE=config.django.devel
+      - DJANGO_LOGGING_FORMATTER=human_readable
+    env_file:
+      - path: ./.env
+        required: false
     ports:
-      - "8080:8000"
+      - "${DJANGO_PORT:-8080}:${DJANGO_PORT:-8080}"
     volumes:
       - "./src/backend:/home/prowler/backend"
       - "./pyproject.toml:/home/prowler/pyproject.toml"
@@ -36,14 +45,17 @@ services:
   postgres:
     image: postgres:16.3-alpine
     ports:
-      - 5432:5432
+      - "${POSTGRES_PORT:-5432}:${POSTGRES_PORT:-5432}"
     hostname: "postgres-db"
     volumes:
       - ./_data/postgres:/var/lib/postgresql/data
     environment:
-      - POSTGRES_PASSWORD=S3cret
+      - POSTGRES_USER=${POSTGRES_USER:-prowler}
-      - POSTGRES_USER=prowler
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-S3cret}
-      - POSTGRES_DB=prowler_db
+      - POSTGRES_DB=${POSTGRES_DB:-prowler_db}
+    env_file:
+      - path: ./.env
+        required: false
     healthcheck:
       test: ["CMD-SHELL", "sh -c 'pg_isready -U prowler -d prowler_db'"]
       interval: 5s

docker-entrypoint.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
+
 apply_migrations() {
   echo "Applying database migrations..."
   poetry run python manage.py migrate
@@ -7,12 +8,12 @@ apply_migrations() {
 
 start_dev_server() {
   echo "Starting the development server..."
-  poetry run python manage.py runserver 0.0.0.0:8000
+  poetry run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}"
 }
 
 start_prod_server() {
   echo "Starting the Gunicorn server..."
-  poetry run gunicorn -c backend/guniconf.py backend.wsgi:application
+  poetry run gunicorn -c config/guniconf.py config.wsgi:application
 }
 
 case "$1" in

poetry.lock

@@ -27,13 +27,13 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"]
 
 [[package]]
 name = "astroid"
-version = "3.2.2"
+version = "3.2.3"
 description = "An abstract syntax tree for Python with inference support."
 optional = false
 python-versions = ">=3.8.0"
 files = [
-    {file = "astroid-3.2.2-py3-none-any.whl", hash = "sha256:e8a0083b4bb28fcffb6207a3bfc9e5d0a68be951dd7e336d5dcf639c682388c0"},
+    {file = "astroid-3.2.3-py3-none-any.whl", hash = "sha256:3eae9ea67c11c858cdd2c91337d2e816bd019ac897ca07d7b346ac10105fceb3"},
-    {file = "astroid-3.2.2.tar.gz", hash = "sha256:8ead48e31b92b2e217b6c9733a21afafe479d52d6e164dd25fb1a770c7c3cf94"},
+    {file = "astroid-3.2.3.tar.gz", hash = "sha256:7099b5a60985529d8d46858befa103b82d0d05a5a5e8b816b5303ed96075e1d9"},
 ]
 
 [[package]]
@@ -460,6 +460,22 @@ files = [
 asgiref = ">=3.6"
 django = ">=3.2"
 
+[[package]]
+name = "django-environ"
+version = "0.11.2"
+description = "A package that allows you to utilize 12factor inspired environment variables to configure your Django application."
+optional = false
+python-versions = ">=3.6,<4"
+files = [
+    {file = "django-environ-0.11.2.tar.gz", hash = "sha256:f32a87aa0899894c27d4e1776fa6b477e8164ed7f6b3e410a62a6d72caaf64be"},
+    {file = "django_environ-0.11.2-py2.py3-none-any.whl", hash = "sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05"},
+]
+
+[package.extras]
+develop = ["coverage[toml] (>=5.0a4)", "furo (>=2021.8.17b43,<2021.9.dev0)", "pytest (>=4.6.11)", "sphinx (>=3.5.0)", "sphinx-notfound-page"]
+docs = ["furo (>=2021.8.17b43,<2021.9.dev0)", "sphinx (>=3.5.0)", "sphinx-notfound-page"]
+testing = ["coverage[toml] (>=5.0a4)", "pytest (>=4.6.11)"]
+
 [[package]]
 name = "django-guid"
 version = "3.5.0"
@@ -1877,4 +1893,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "ea45391b9e5854229550df13e428458b33bf5a31acc16c1ef9d9e997647b4800"
+content-hash = "76aaacac73e4a835736c1c6550809e09481e405974e8ad89699cbcb52bcbe0e0"

pyproject.toml

@@ -13,6 +13,7 @@ version = "1.0.0"
 [tool.poetry.dependencies]
 django = "5.0.7"
 django-cors-headers = "4.4.0"
+django-environ = "0.11.2"
 django-guid = "3.5.0"
 djangorestframework = "3.15.2"
 drf-spectacular = "0.27.2"

src/backend/api/middleware.py

@@ -1,7 +1,7 @@
 import logging
 import time
 
-from backend.custom_logging import BackendLogger
+from config.custom_logging import BackendLogger
 
 
 class APILoggingMiddleware:

src/backend/config/asgi.py

@@ -11,6 +11,6 @@ import os
 
 from django.core.asgi import get_asgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
 
 application = get_asgi_application()

src/backend/config/custom_logging.py

@@ -4,6 +4,8 @@ from enum import StrEnum
 
 from django_guid.log_filters import CorrelationId
 
+from config.env import env
+
 
 class BackendLogger(StrEnum):
     GUNICORN = "gunicorn"
@@ -115,6 +117,9 @@ class TransactionIdFilter(CorrelationId):
 
 # Logging settings
 
+LEVEL = env("DJANGO_LOGGING_LEVEL", default="INFO")
+FORMATTER = env("DJANGO_LOGGING_FORMATTER", default="ndjson")
+
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": True,
@@ -128,70 +133,49 @@ LOGGING = {
             "()": HumanReadableFormatter,
             "datefmt": "%Y-%m-%d %H:%M:%S",
         },
-        "django": {
-            "format": "{asctime} [{name}] {levelname}: ({module}) {message}",
-            "style": "{",
-        },
-        "api": {
-            "format": '{asctime} [{name}] {levelname}: "{method} {path}" with parameters {query_params} done in '
-            "{duration}s: {status_code}",
-            "style": "{",
-        },
-        "db": {
-            "format": "{asctime} [{name}] {levelname}: {message}",
-            "style": "{",
-        },
-        "security": {
-            "format": "{asctime} [{name}] {levelname}: {message}",
-            "style": "{",
-        },
-        "tasks": {
-            "format": "{asctime} [{name}] {levelname}: {message}",
-            "style": "{",
-        },
     },
     "handlers": {
         "gunicorn_console": {
-            "level": "INFO",
+            "level": LEVEL,
             "class": "logging.StreamHandler",
-            "formatter": "human_readable",
+            "formatter": FORMATTER,
             "filters": ["transaction_id"],
         },
         "django_console": {
-            "level": "INFO",
+            "level": LEVEL,
             "class": "logging.StreamHandler",
-            "formatter": "human_readable",
+            "formatter": FORMATTER,
             "filters": ["transaction_id"],
         },
         "api_console": {
-            "level": "INFO",
+            "level": LEVEL,
             "class": "logging.StreamHandler",
-            "formatter": "human_readable",
+            "formatter": FORMATTER,
             "filters": ["transaction_id"],
         },
         "db_console": {
-            "level": "DEBUG",
+            "level": f"{'DEBUG' if LEVEL == 'DEBUG' else 'INFO'}",
             "class": "logging.StreamHandler",
-            "formatter": "human_readable",
+            "formatter": FORMATTER,
             "filters": ["transaction_id"],
         },
         "security_console": {
-            "level": "INFO",
+            "level": LEVEL,
             "class": "logging.StreamHandler",
-            "formatter": "human_readable",
+            "formatter": FORMATTER,
             "filters": ["transaction_id"],
         },
         "tasks_console": {
-            "level": "INFO",
+            "level": LEVEL,
             "class": "logging.StreamHandler",
-            "formatter": "human_readable",
+            "formatter": FORMATTER,
             "filters": ["transaction_id"],
         },
     },
     "loggers": {
         BackendLogger.GUNICORN: {
             "handlers": ["gunicorn_console"],
-            "level": "INFO",
+            "level": LEVEL,
             "propagate": False,
         },
         BackendLogger.GUNICORN_ACCESS: {
@@ -201,7 +185,7 @@ LOGGING = {
         },
         BackendLogger.GUNICORN_ERROR: {
             "handlers": ["gunicorn_console"],
-            "level": "INFO",
+            "level": LEVEL,
             "propagate": False,
         },
         BackendLogger.DJANGO: {
@@ -211,22 +195,22 @@ LOGGING = {
         },
         BackendLogger.DB: {
             "handlers": ["db_console"],
-            "level": "INFO",
+            "level": LEVEL,
             "propagate": False,
         },
         BackendLogger.SECURITY: {
             "handlers": ["security_console"],
-            "level": "INFO",
+            "level": LEVEL,
             "propagate": False,
         },
         BackendLogger.API: {
             "handlers": ["api_console"],
-            "level": "INFO",
+            "level": LEVEL,
             "propagate": False,
         },
         BackendLogger.TASKS: {
             "handlers": ["tasks_console"],
-            "level": "INFO",
+            "level": LEVEL,
             "propagate": False,
         },
     },

src/backend/config/django/base.py

@@ -1,21 +1,10 @@
-from pathlib import Path
+from config.custom_logging import LOGGING  # noqa
-
+from config.env import BASE_DIR, env  # noqa
-from backend.custom_logging import LOGGING  # noqa
-
-# Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent
 
 
-# Quick-start development settings - unsuitable for production
+SECRET_KEY = env("SECRET_KEY", default="secret")
-# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/
+DEBUG = env.bool("DJANGO_DEBUG", default=False)
-
+ALLOWED_HOSTS = ["localhost", "127.0.0.1"]
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "django-insecure-9ut_dim+ma+r=ys094hoiwns#g1t@0*_&-*d!g+h)y%lye$0_^"
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = ["*"]
 
 
 # Application definition
@@ -49,7 +38,7 @@ MIDDLEWARE = [
 
 CORS_ALLOWED_ORIGINS = ["http://localhost", "http://127.0.0.1"]
 
-ROOT_URLCONF = "backend.urls"
+ROOT_URLCONF = "config.urls"
 
 TEMPLATES = [
     {
@@ -78,20 +67,7 @@ SPECTACULAR_SETTINGS = {
     "SERVE_INCLUDE_SCHEMA": True,
 }
 
-WSGI_APPLICATION = "backend.wsgi.application"
+WSGI_APPLICATION = "config.wsgi.application"
-
-# Database
-
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.postgresql",
-        "NAME": "prowler_db",
-        "USER": "prowler",
-        "PASSWORD": "S3cret",
-        "HOST": "postgres-db",
-        "PORT": "5432",
-    }
-}
 
 DJANGO_GUID = {
     "GUID_HEADER_NAME": "Transaction-ID",

src/backend/config/django/devel.py

@@ -0,0 +1,18 @@
+from config.django.base import *  # noqa
+from config.env import env
+
+
+DEBUG = env.bool("DJANGO_DEBUG", default=True)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"])
+
+# Database
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": env("POSTGRES_DB", default="prowler_db"),
+        "USER": env("POSTGRES_USER", default="prowler"),
+        "PASSWORD": env("POSTGRES_PASSWORD", default="S3cret"),
+        "HOST": env("POSTGRES_HOST", default="postgres-db"),
+        "PORT": env("POSTGRES_PORT", default="5432"),
+    }
+}

src/backend/config/django/production.py

@@ -0,0 +1,18 @@
+from config.django.base import *  # noqa
+from config.env import env
+
+
+DEBUG = env.bool("DJANGO_DEBUG", default=False)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
+
+# Database
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": env("POSTGRES_DB"),
+        "USER": env("POSTGRES_USER"),
+        "PASSWORD": env("POSTGRES_PASSWORD"),
+        "HOST": env("POSTGRES_HOST"),
+        "PORT": env("POSTGRES_PORT"),
+    }
+}

src/backend/config/django/testing.py

@@ -0,0 +1,14 @@
+from config.django.base import *  # noqa
+from config.env import env
+
+
+DEBUG = env.bool("DJANGO_DEBUG", default=False)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
+
+# Database
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": ":memory:",
+    }
+}

src/backend/config/env.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+import environ
+
+env = environ.Env()
+
+BASE_DIR = Path(__file__).resolve().parent.parent

src/backend/config/guniconf.py

@@ -2,21 +2,26 @@ import logging
 import multiprocessing
 import os
 
+from config.env import env
+
 # Ensure the environment variable for Django settings is set
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
 
 # Import Django and set it up before accessing settings
 import django  # noqa: E402
 
 django.setup()
-from backend.settings import LOGGING as DJANGO_LOGGERS  # noqa: E402
+from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG  # noqa: E402
-from backend.custom_logging import BackendLogger  # noqa: E402
+from config.custom_logging import BackendLogger  # noqa: E402
+
+BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1")
+PORT = env("DJANGO_PORT", default=8000)
 
 # Server settings
-bind = ["0.0.0.0:8000"]
+bind = f"{BIND_ADDRESS}:{PORT}"
-# Calculate the number of workers based on CPU count. Refactor when adding config from env vars
+
-workers = multiprocessing.cpu_count() * 2 + 1
+workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1)
-reload = True
+reload = DEBUG
 
 # Logging
 logconfig_dict = DJANGO_LOGGERS

src/backend/config/wsgi.py

@@ -11,6 +11,6 @@ import os
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
 
 application = get_wsgi_application()

src/backend/manage.py

@@ -7,7 +7,8 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
+
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

src/backend/pytest.ini

@@ -1,3 +1,3 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = backend.settings
+DJANGO_SETTINGS_MODULE = config.django.testing
 addopts = -rP