chore: PRWLR-4055 Update project structure and add env vars (#12)

* chore: PRWLR-4055 update project structure and add env vars

* chore: PRWLR-4055 make production settings default

* chore: PRWLR-4055 change default values to env variables

* chore: PRWLR-4055 apply requested changes on default values

* chore: PRWLR-4055 adapt testing environment

src/backend/config/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for backend project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
+
+application = get_asgi_application()

src/backend/config/custom_logging.py

@@ -0,0 +1,222 @@
+import json
+import logging
+from enum import StrEnum
+
+from django_guid.log_filters import CorrelationId
+
+from config.env import env
+
+
+class BackendLogger(StrEnum):
+    GUNICORN = "gunicorn"
+    GUNICORN_ACCESS = "gunicorn.access"
+    GUNICORN_ERROR = "gunicorn.error"
+    DJANGO = "django"
+    SECURITY = "django.security"
+    DB = "django.db"
+    API = "api"
+    TASKS = "tasks"
+
+
+# Formatters
+
+
+class NDJSONFormatter(logging.Formatter):
+    """NDJSON custom formatter for logging messages.
+
+    If available, it will include all kind of API request metadata.
+    """
+
+    def format(self, record):
+        log_record = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "message": record.getMessage(),
+            "logger": record.name,
+            "module": record.module,
+            "pathname": record.pathname,
+            "lineno": record.lineno,
+            "funcName": record.funcName,
+            "process": record.process,
+            "thread": record.thread,
+            "transaction_id": record.transaction_id
+            if hasattr(record, "transaction_id")
+            else None,
+        }
+
+        # Add REST API extra fields
+        if hasattr(record, "method"):
+            log_record["method"] = record.method
+        if hasattr(record, "path"):
+            log_record["path"] = record.path
+        if hasattr(record, "query_params"):
+            log_record["query_params"] = record.query_params
+        if hasattr(record, "duration"):
+            log_record["duration"] = record.duration
+        if hasattr(record, "status_code"):
+            log_record["status_code"] = record.status_code
+
+        if record.exc_info:
+            log_record["exc_info"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_record)
+
+
+class HumanReadableFormatter(logging.Formatter):
+    """Human-readable custom formatter for logging messages.
+
+    If available, it will include all kinds of API request metadata.
+    """
+
+    def format(self, record):
+        log_components = [
+            f"{self.formatTime(record, self.datefmt)}",
+            f"[{record.name}]",
+            f"{record.levelname}:",
+            f"({record.module})",
+            f"[module={record.module}",
+            f"path={record.pathname}",
+            f"line={record.lineno}",
+            f"function={record.funcName}",
+            f"process={record.process}",
+            f"thread={record.thread}",
+            f"transaction-id={record.transaction_id if hasattr(record, "transaction_id") else None}]",
+            f"{record.getMessage()}",
+        ]
+
+        # Add REST API extra fields
+        if hasattr(record, "method"):
+            log_components.append(f'"{record.method} {record.path}"')
+        if hasattr(record, "query_params"):
+            log_components.append(f"with parameters {record.query_params}")
+        if hasattr(record, "duration"):
+            log_components.append(f"done in {record.duration}s:")
+        if hasattr(record, "status_code"):
+            log_components.append(f"{record.status_code}")
+
+        if record.exc_info:
+            log_components.append(self.formatException(record.exc_info))
+
+        return " ".join(log_components)
+
+
+# Filters
+
+
+class TransactionIdFilter(CorrelationId):
+    """Logging filter class.
+
+    Used to override the `correlation_id_field` parameter in the parent class to use a different name.
+    """
+
+    CORRELATION_ID_FIELD = "transaction_id"
+
+    def __init__(self):
+        super().__init__(correlation_id_field=self.CORRELATION_ID_FIELD)
+
+
+# Logging settings
+
+LEVEL = env("DJANGO_LOGGING_LEVEL", default="INFO")
+FORMATTER = env("DJANGO_LOGGING_FORMATTER", default="ndjson")
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": True,
+    "filters": {"transaction_id": {"()": TransactionIdFilter}},
+    "formatters": {
+        "ndjson": {
+            "()": NDJSONFormatter,
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
+        "human_readable": {
+            "()": HumanReadableFormatter,
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
+    },
+    "handlers": {
+        "gunicorn_console": {
+            "level": LEVEL,
+            "class": "logging.StreamHandler",
+            "formatter": FORMATTER,
+            "filters": ["transaction_id"],
+        },
+        "django_console": {
+            "level": LEVEL,
+            "class": "logging.StreamHandler",
+            "formatter": FORMATTER,
+            "filters": ["transaction_id"],
+        },
+        "api_console": {
+            "level": LEVEL,
+            "class": "logging.StreamHandler",
+            "formatter": FORMATTER,
+            "filters": ["transaction_id"],
+        },
+        "db_console": {
+            "level": f"{'DEBUG' if LEVEL == 'DEBUG' else 'INFO'}",
+            "class": "logging.StreamHandler",
+            "formatter": FORMATTER,
+            "filters": ["transaction_id"],
+        },
+        "security_console": {
+            "level": LEVEL,
+            "class": "logging.StreamHandler",
+            "formatter": FORMATTER,
+            "filters": ["transaction_id"],
+        },
+        "tasks_console": {
+            "level": LEVEL,
+            "class": "logging.StreamHandler",
+            "formatter": FORMATTER,
+            "filters": ["transaction_id"],
+        },
+    },
+    "loggers": {
+        BackendLogger.GUNICORN: {
+            "handlers": ["gunicorn_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        BackendLogger.GUNICORN_ACCESS: {
+            "handlers": ["gunicorn_console"],
+            "level": "CRITICAL",
+            "propagate": False,
+        },
+        BackendLogger.GUNICORN_ERROR: {
+            "handlers": ["gunicorn_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        BackendLogger.DJANGO: {
+            "handlers": ["django_console"],
+            "level": "WARNING",
+            "propagate": True,
+        },
+        BackendLogger.DB: {
+            "handlers": ["db_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        BackendLogger.SECURITY: {
+            "handlers": ["security_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        BackendLogger.API: {
+            "handlers": ["api_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+        BackendLogger.TASKS: {
+            "handlers": ["tasks_console"],
+            "level": LEVEL,
+            "propagate": False,
+        },
+    },
+    # Gunicorn required configuration
+    "root": {
+        "level": "ERROR",
+        "handlers": ["gunicorn_console"],
+    },
+}

src/backend/config/django/base.py

@@ -0,0 +1,124 @@
+from config.custom_logging import LOGGING  # noqa
+from config.env import BASE_DIR, env  # noqa
+
+
+SECRET_KEY = env("SECRET_KEY", default="secret")
+DEBUG = env.bool("DJANGO_DEBUG", default=False)
+ALLOWED_HOSTS = ["localhost", "127.0.0.1"]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "api",
+    "rest_framework",
+    "corsheaders",
+    "drf_spectacular",
+    "django_guid",
+]
+
+MIDDLEWARE = [
+    "django_guid.middleware.guid_middleware",
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "corsheaders.middleware.CorsMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "api.middleware.APILoggingMiddleware",
+]
+
+CORS_ALLOWED_ORIGINS = ["http://localhost", "http://127.0.0.1"]
+
+ROOT_URLCONF = "config.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+REST_FRAMEWORK = {
+    "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
+}
+
+SPECTACULAR_SETTINGS = {
+    "TITLE": "Prowler RESTful API",
+    "DESCRIPTION": "Prowler RESTful API specification.\n\nThis file is auto-generated.",
+    "VERSION": "1.0.0",
+    "SERVE_INCLUDE_SCHEMA": True,
+}
+
+WSGI_APPLICATION = "config.wsgi.application"
+
+DJANGO_GUID = {
+    "GUID_HEADER_NAME": "Transaction-ID",
+    "VALIDATE_GUID": True,
+    "RETURN_HEADER": True,
+    "EXPOSE_HEADER": True,
+    "INTEGRATIONS": [],
+    "IGNORE_URLS": [],
+    "UUID_LENGTH": 32,
+}
+
+# Password validation
+# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.0/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+LANGUAGES = [
+    ("en", "English"),
+]
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.0/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

src/backend/config/django/devel.py

@@ -0,0 +1,18 @@
+from config.django.base import *  # noqa
+from config.env import env
+
+
+DEBUG = env.bool("DJANGO_DEBUG", default=True)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"])
+
+# Database
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": env("POSTGRES_DB", default="prowler_db"),
+        "USER": env("POSTGRES_USER", default="prowler"),
+        "PASSWORD": env("POSTGRES_PASSWORD", default="S3cret"),
+        "HOST": env("POSTGRES_HOST", default="postgres-db"),
+        "PORT": env("POSTGRES_PORT", default="5432"),
+    }
+}

src/backend/config/django/production.py

@@ -0,0 +1,18 @@
+from config.django.base import *  # noqa
+from config.env import env
+
+
+DEBUG = env.bool("DJANGO_DEBUG", default=False)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
+
+# Database
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": env("POSTGRES_DB"),
+        "USER": env("POSTGRES_USER"),
+        "PASSWORD": env("POSTGRES_PASSWORD"),
+        "HOST": env("POSTGRES_HOST"),
+        "PORT": env("POSTGRES_PORT"),
+    }
+}

src/backend/config/django/testing.py

@@ -0,0 +1,14 @@
+from config.django.base import *  # noqa
+from config.env import env
+
+
+DEBUG = env.bool("DJANGO_DEBUG", default=False)
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
+
+# Database
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": ":memory:",
+    }
+}

src/backend/config/env.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+import environ
+
+env = environ.Env()
+
+BASE_DIR = Path(__file__).resolve().parent.parent

src/backend/config/guniconf.py

@@ -0,0 +1,43 @@
+import logging
+import multiprocessing
+import os
+
+from config.env import env
+
+# Ensure the environment variable for Django settings is set
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
+
+# Import Django and set it up before accessing settings
+import django  # noqa: E402
+
+django.setup()
+from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG  # noqa: E402
+from config.custom_logging import BackendLogger  # noqa: E402
+
+BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1")
+PORT = env("DJANGO_PORT", default=8000)
+
+# Server settings
+bind = f"{BIND_ADDRESS}:{PORT}"
+
+workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1)
+reload = DEBUG
+
+# Logging
+logconfig_dict = DJANGO_LOGGERS
+gunicorn_logger = logging.getLogger(BackendLogger.GUNICORN)
+
+
+# Hooks
+def on_starting(_):
+    gunicorn_logger.info(f"Starting gunicorn server with {workers} workers")
+    if reload:
+        gunicorn_logger.warning("Reload settings enabled (dev mode)")
+
+
+def on_reload(_):
+    gunicorn_logger.warning("Gunicorn server has reloaded")
+
+
+def when_ready(_):
+    gunicorn_logger.info("Gunicorn server is ready")

src/backend/config/urls.py

@@ -0,0 +1,7 @@
+from django.contrib import admin
+from django.urls import include, path
+
+urlpatterns = [
+    path("admin/", admin.site.urls),
+    path("api/v1/", include("api.urls.v1.urls")),
+]

src/backend/config/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for backend project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
+
+application = get_wsgi_application()