From 8f41b38bbf800dc06a67f34c422f0dcf5ced8679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Fern=C3=A1ndez=20Poyatos?= Date: Tue, 16 Jul 2024 15:33:56 +0200 Subject: [PATCH] chore: PRWLR-4055 Update project structure and add env vars (#12) * chore: PRWLR-4055 update project structure and add env vars * chore: PRWLR-4055 make production settings default * chore: PRWLR-4055 change default values to env variables * chore: PRWLR-4055 apply requested changes on default values * chore: PRWLR-4055 adapt testing environment --- .env.example | 15 +++++ docker-compose.yml | 24 +++++-- docker-entrypoint.sh | 5 +- poetry.lock | 24 +++++-- pyproject.toml | 1 + src/backend/api/middleware.py | 2 +- src/backend/{backend => config}/__init__.py | 0 src/backend/{backend => config}/asgi.py | 2 +- .../{backend => config}/custom_logging.py | 62 +++++++------------ src/backend/config/django/__init__.py | 0 .../settings.py => config/django/base.py} | 38 +++--------- src/backend/config/django/devel.py | 18 ++++++ src/backend/config/django/production.py | 18 ++++++ src/backend/config/django/testing.py | 14 +++++ src/backend/config/env.py | 7 +++ src/backend/{backend => config}/guniconf.py | 19 +++--- src/backend/config/settings/__init__.py | 0 src/backend/{backend => config}/urls.py | 0 src/backend/{backend => config}/wsgi.py | 2 +- src/backend/manage.py | 3 +- src/backend/pytest.ini | 2 +- 21 files changed, 162 insertions(+), 94 deletions(-) create mode 100644 .env.example rename src/backend/{backend => config}/__init__.py (100%) rename src/backend/{backend => config}/asgi.py (80%) rename src/backend/{backend => config}/custom_logging.py (81%) create mode 100644 src/backend/config/django/__init__.py rename src/backend/{backend/settings.py => config/django/base.py} (76%) create mode 100644 src/backend/config/django/devel.py create mode 100644 src/backend/config/django/production.py create mode 100644 src/backend/config/django/testing.py create mode 100644 src/backend/config/env.py rename src/backend/{backend => config}/guniconf.py (58%) create mode 100644 src/backend/config/settings/__init__.py rename src/backend/{backend => config}/urls.py (100%) rename src/backend/{backend => config}/wsgi.py (80%) diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000..7cda3e66d1 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Django settings +DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1 +DJANGO_BIND_ADDRESS=0.0.0.0 +DJANGO_PORT=8000 +DJANGO_DEBUG=False +DJANGO_SETTINGS_MODULE=config.django.[production|devel] +DJANGO_LOGGING_FORMATTER=[ndjson|human_readable] +DJANGO_WORKERS=4 # Defaults to the maximum available based on CPU cores if not set. + +# PostgreSQL settings +POSTGRES_HOST=postgres-db +POSTGRES_PORT=5432 +POSTGRES_USER=prowler +POSTGRES_PASSWORD=S3cret +POSTGRES_DB=prowler_db diff --git a/docker-compose.yml b/docker-compose.yml index d3a5fefc47..49752e15ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,8 +3,11 @@ services: build: dockerfile: Dockerfile image: prowler-restful-api + env_file: + - path: ./.env + required: false ports: - - "8000:8000" + - "${DJANGO_PORT:-8000}:${DJANGO_PORT:-8000}" profiles: - prod depends_on: @@ -19,8 +22,14 @@ services: dockerfile: Dockerfile target: dev image: prowler-restful-api-dev + environment: + - DJANGO_SETTINGS_MODULE=config.django.devel + - DJANGO_LOGGING_FORMATTER=human_readable + env_file: + - path: ./.env + required: false ports: - - "8080:8000" + - "${DJANGO_PORT:-8080}:${DJANGO_PORT:-8080}" volumes: - "./src/backend:/home/prowler/backend" - "./pyproject.toml:/home/prowler/pyproject.toml" @@ -36,14 +45,17 @@ services: postgres: image: postgres:16.3-alpine ports: - - 5432:5432 + - "${POSTGRES_PORT:-5432}:${POSTGRES_PORT:-5432}" hostname: "postgres-db" volumes: - ./_data/postgres:/var/lib/postgresql/data environment: - - POSTGRES_PASSWORD=S3cret - - POSTGRES_USER=prowler - - POSTGRES_DB=prowler_db + - POSTGRES_USER=${POSTGRES_USER:-prowler} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-S3cret} + - POSTGRES_DB=${POSTGRES_DB:-prowler_db} + env_file: + - path: ./.env + required: false healthcheck: test: ["CMD-SHELL", "sh -c 'pg_isready -U prowler -d prowler_db'"] interval: 5s diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index b3c5926a12..9089e15946 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,5 +1,6 @@ #!/bin/sh + apply_migrations() { echo "Applying database migrations..." poetry run python manage.py migrate @@ -7,12 +8,12 @@ apply_migrations() { start_dev_server() { echo "Starting the development server..." - poetry run python manage.py runserver 0.0.0.0:8000 + poetry run python manage.py runserver 0.0.0.0:"${DJANGO_PORT:-8080}" } start_prod_server() { echo "Starting the Gunicorn server..." - poetry run gunicorn -c backend/guniconf.py backend.wsgi:application + poetry run gunicorn -c config/guniconf.py config.wsgi:application } case "$1" in diff --git a/poetry.lock b/poetry.lock index 03d634a125..bfd3e412c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -27,13 +27,13 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] [[package]] name = "astroid" -version = "3.2.2" +version = "3.2.3" description = "An abstract syntax tree for Python with inference support." optional = false python-versions = ">=3.8.0" files = [ - {file = "astroid-3.2.2-py3-none-any.whl", hash = "sha256:e8a0083b4bb28fcffb6207a3bfc9e5d0a68be951dd7e336d5dcf639c682388c0"}, - {file = "astroid-3.2.2.tar.gz", hash = "sha256:8ead48e31b92b2e217b6c9733a21afafe479d52d6e164dd25fb1a770c7c3cf94"}, + {file = "astroid-3.2.3-py3-none-any.whl", hash = "sha256:3eae9ea67c11c858cdd2c91337d2e816bd019ac897ca07d7b346ac10105fceb3"}, + {file = "astroid-3.2.3.tar.gz", hash = "sha256:7099b5a60985529d8d46858befa103b82d0d05a5a5e8b816b5303ed96075e1d9"}, ] [[package]] @@ -460,6 +460,22 @@ files = [ asgiref = ">=3.6" django = ">=3.2" +[[package]] +name = "django-environ" +version = "0.11.2" +description = "A package that allows you to utilize 12factor inspired environment variables to configure your Django application." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "django-environ-0.11.2.tar.gz", hash = "sha256:f32a87aa0899894c27d4e1776fa6b477e8164ed7f6b3e410a62a6d72caaf64be"}, + {file = "django_environ-0.11.2-py2.py3-none-any.whl", hash = "sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05"}, +] + +[package.extras] +develop = ["coverage[toml] (>=5.0a4)", "furo (>=2021.8.17b43,<2021.9.dev0)", "pytest (>=4.6.11)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] +docs = ["furo (>=2021.8.17b43,<2021.9.dev0)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] +testing = ["coverage[toml] (>=5.0a4)", "pytest (>=4.6.11)"] + [[package]] name = "django-guid" version = "3.5.0" @@ -1877,4 +1893,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "ea45391b9e5854229550df13e428458b33bf5a31acc16c1ef9d9e997647b4800" +content-hash = "76aaacac73e4a835736c1c6550809e09481e405974e8ad89699cbcb52bcbe0e0" diff --git a/pyproject.toml b/pyproject.toml index 1ec9521471..58143e0b42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ version = "1.0.0" [tool.poetry.dependencies] django = "5.0.7" django-cors-headers = "4.4.0" +django-environ = "0.11.2" django-guid = "3.5.0" djangorestframework = "3.15.2" drf-spectacular = "0.27.2" diff --git a/src/backend/api/middleware.py b/src/backend/api/middleware.py index c301cf1c6a..cad6e8c8cb 100644 --- a/src/backend/api/middleware.py +++ b/src/backend/api/middleware.py @@ -1,7 +1,7 @@ import logging import time -from backend.custom_logging import BackendLogger +from config.custom_logging import BackendLogger class APILoggingMiddleware: diff --git a/src/backend/backend/__init__.py b/src/backend/config/__init__.py similarity index 100% rename from src/backend/backend/__init__.py rename to src/backend/config/__init__.py diff --git a/src/backend/backend/asgi.py b/src/backend/config/asgi.py similarity index 80% rename from src/backend/backend/asgi.py rename to src/backend/config/asgi.py index 0e40a3e25b..cfc064ba97 100644 --- a/src/backend/backend/asgi.py +++ b/src/backend/config/asgi.py @@ -11,6 +11,6 @@ import os from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production") application = get_asgi_application() diff --git a/src/backend/backend/custom_logging.py b/src/backend/config/custom_logging.py similarity index 81% rename from src/backend/backend/custom_logging.py rename to src/backend/config/custom_logging.py index 24462bbca6..6e869db026 100644 --- a/src/backend/backend/custom_logging.py +++ b/src/backend/config/custom_logging.py @@ -4,6 +4,8 @@ from enum import StrEnum from django_guid.log_filters import CorrelationId +from config.env import env + class BackendLogger(StrEnum): GUNICORN = "gunicorn" @@ -115,6 +117,9 @@ class TransactionIdFilter(CorrelationId): # Logging settings +LEVEL = env("DJANGO_LOGGING_LEVEL", default="INFO") +FORMATTER = env("DJANGO_LOGGING_FORMATTER", default="ndjson") + LOGGING = { "version": 1, "disable_existing_loggers": True, @@ -128,70 +133,49 @@ LOGGING = { "()": HumanReadableFormatter, "datefmt": "%Y-%m-%d %H:%M:%S", }, - "django": { - "format": "{asctime} [{name}] {levelname}: ({module}) {message}", - "style": "{", - }, - "api": { - "format": '{asctime} [{name}] {levelname}: "{method} {path}" with parameters {query_params} done in ' - "{duration}s: {status_code}", - "style": "{", - }, - "db": { - "format": "{asctime} [{name}] {levelname}: {message}", - "style": "{", - }, - "security": { - "format": "{asctime} [{name}] {levelname}: {message}", - "style": "{", - }, - "tasks": { - "format": "{asctime} [{name}] {levelname}: {message}", - "style": "{", - }, }, "handlers": { "gunicorn_console": { - "level": "INFO", + "level": LEVEL, "class": "logging.StreamHandler", - "formatter": "human_readable", + "formatter": FORMATTER, "filters": ["transaction_id"], }, "django_console": { - "level": "INFO", + "level": LEVEL, "class": "logging.StreamHandler", - "formatter": "human_readable", + "formatter": FORMATTER, "filters": ["transaction_id"], }, "api_console": { - "level": "INFO", + "level": LEVEL, "class": "logging.StreamHandler", - "formatter": "human_readable", + "formatter": FORMATTER, "filters": ["transaction_id"], }, "db_console": { - "level": "DEBUG", + "level": f"{'DEBUG' if LEVEL == 'DEBUG' else 'INFO'}", "class": "logging.StreamHandler", - "formatter": "human_readable", + "formatter": FORMATTER, "filters": ["transaction_id"], }, "security_console": { - "level": "INFO", + "level": LEVEL, "class": "logging.StreamHandler", - "formatter": "human_readable", + "formatter": FORMATTER, "filters": ["transaction_id"], }, "tasks_console": { - "level": "INFO", + "level": LEVEL, "class": "logging.StreamHandler", - "formatter": "human_readable", + "formatter": FORMATTER, "filters": ["transaction_id"], }, }, "loggers": { BackendLogger.GUNICORN: { "handlers": ["gunicorn_console"], - "level": "INFO", + "level": LEVEL, "propagate": False, }, BackendLogger.GUNICORN_ACCESS: { @@ -201,7 +185,7 @@ LOGGING = { }, BackendLogger.GUNICORN_ERROR: { "handlers": ["gunicorn_console"], - "level": "INFO", + "level": LEVEL, "propagate": False, }, BackendLogger.DJANGO: { @@ -211,22 +195,22 @@ LOGGING = { }, BackendLogger.DB: { "handlers": ["db_console"], - "level": "INFO", + "level": LEVEL, "propagate": False, }, BackendLogger.SECURITY: { "handlers": ["security_console"], - "level": "INFO", + "level": LEVEL, "propagate": False, }, BackendLogger.API: { "handlers": ["api_console"], - "level": "INFO", + "level": LEVEL, "propagate": False, }, BackendLogger.TASKS: { "handlers": ["tasks_console"], - "level": "INFO", + "level": LEVEL, "propagate": False, }, }, diff --git a/src/backend/config/django/__init__.py b/src/backend/config/django/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/backend/backend/settings.py b/src/backend/config/django/base.py similarity index 76% rename from src/backend/backend/settings.py rename to src/backend/config/django/base.py index b0d6a99d99..47d7d74f10 100644 --- a/src/backend/backend/settings.py +++ b/src/backend/config/django/base.py @@ -1,21 +1,10 @@ -from pathlib import Path - -from backend.custom_logging import LOGGING # noqa - -# Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent +from config.custom_logging import LOGGING # noqa +from config.env import BASE_DIR, env # noqa -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "django-insecure-9ut_dim+ma+r=ys094hoiwns#g1t@0*_&-*d!g+h)y%lye$0_^" - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = ["*"] +SECRET_KEY = env("SECRET_KEY", default="secret") +DEBUG = env.bool("DJANGO_DEBUG", default=False) +ALLOWED_HOSTS = ["localhost", "127.0.0.1"] # Application definition @@ -49,7 +38,7 @@ MIDDLEWARE = [ CORS_ALLOWED_ORIGINS = ["http://localhost", "http://127.0.0.1"] -ROOT_URLCONF = "backend.urls" +ROOT_URLCONF = "config.urls" TEMPLATES = [ { @@ -78,20 +67,7 @@ SPECTACULAR_SETTINGS = { "SERVE_INCLUDE_SCHEMA": True, } -WSGI_APPLICATION = "backend.wsgi.application" - -# Database - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "prowler_db", - "USER": "prowler", - "PASSWORD": "S3cret", - "HOST": "postgres-db", - "PORT": "5432", - } -} +WSGI_APPLICATION = "config.wsgi.application" DJANGO_GUID = { "GUID_HEADER_NAME": "Transaction-ID", diff --git a/src/backend/config/django/devel.py b/src/backend/config/django/devel.py new file mode 100644 index 0000000000..a03daf4d0b --- /dev/null +++ b/src/backend/config/django/devel.py @@ -0,0 +1,18 @@ +from config.django.base import * # noqa +from config.env import env + + +DEBUG = env.bool("DJANGO_DEBUG", default=True) +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"]) + +# Database +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": env("POSTGRES_DB", default="prowler_db"), + "USER": env("POSTGRES_USER", default="prowler"), + "PASSWORD": env("POSTGRES_PASSWORD", default="S3cret"), + "HOST": env("POSTGRES_HOST", default="postgres-db"), + "PORT": env("POSTGRES_PORT", default="5432"), + } +} diff --git a/src/backend/config/django/production.py b/src/backend/config/django/production.py new file mode 100644 index 0000000000..bf99d9ad19 --- /dev/null +++ b/src/backend/config/django/production.py @@ -0,0 +1,18 @@ +from config.django.base import * # noqa +from config.env import env + + +DEBUG = env.bool("DJANGO_DEBUG", default=False) +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"]) + +# Database +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": env("POSTGRES_DB"), + "USER": env("POSTGRES_USER"), + "PASSWORD": env("POSTGRES_PASSWORD"), + "HOST": env("POSTGRES_HOST"), + "PORT": env("POSTGRES_PORT"), + } +} diff --git a/src/backend/config/django/testing.py b/src/backend/config/django/testing.py new file mode 100644 index 0000000000..07326f7010 --- /dev/null +++ b/src/backend/config/django/testing.py @@ -0,0 +1,14 @@ +from config.django.base import * # noqa +from config.env import env + + +DEBUG = env.bool("DJANGO_DEBUG", default=False) +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"]) + +# Database +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + } +} diff --git a/src/backend/config/env.py b/src/backend/config/env.py new file mode 100644 index 0000000000..55ec0b9718 --- /dev/null +++ b/src/backend/config/env.py @@ -0,0 +1,7 @@ +from pathlib import Path + +import environ + +env = environ.Env() + +BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/src/backend/backend/guniconf.py b/src/backend/config/guniconf.py similarity index 58% rename from src/backend/backend/guniconf.py rename to src/backend/config/guniconf.py index 7011322726..a5b625874b 100644 --- a/src/backend/backend/guniconf.py +++ b/src/backend/config/guniconf.py @@ -2,21 +2,26 @@ import logging import multiprocessing import os +from config.env import env + # Ensure the environment variable for Django settings is set -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production") # Import Django and set it up before accessing settings import django # noqa: E402 django.setup() -from backend.settings import LOGGING as DJANGO_LOGGERS # noqa: E402 -from backend.custom_logging import BackendLogger # noqa: E402 +from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG # noqa: E402 +from config.custom_logging import BackendLogger # noqa: E402 + +BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1") +PORT = env("DJANGO_PORT", default=8000) # Server settings -bind = ["0.0.0.0:8000"] -# Calculate the number of workers based on CPU count. Refactor when adding config from env vars -workers = multiprocessing.cpu_count() * 2 + 1 -reload = True +bind = f"{BIND_ADDRESS}:{PORT}" + +workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1) +reload = DEBUG # Logging logconfig_dict = DJANGO_LOGGERS diff --git a/src/backend/config/settings/__init__.py b/src/backend/config/settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/backend/backend/urls.py b/src/backend/config/urls.py similarity index 100% rename from src/backend/backend/urls.py rename to src/backend/config/urls.py diff --git a/src/backend/backend/wsgi.py b/src/backend/config/wsgi.py similarity index 80% rename from src/backend/backend/wsgi.py rename to src/backend/config/wsgi.py index fe4a826a92..6c770033c4 100644 --- a/src/backend/backend/wsgi.py +++ b/src/backend/config/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production") application = get_wsgi_application() diff --git a/src/backend/manage.py b/src/backend/manage.py index ae97db8bae..590fbeb713 100755 --- a/src/backend/manage.py +++ b/src/backend/manage.py @@ -7,7 +7,8 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/src/backend/pytest.ini b/src/backend/pytest.ini index 9e4d4a48a7..ebce5cf212 100644 --- a/src/backend/pytest.ini +++ b/src/backend/pytest.ini @@ -1,3 +1,3 @@ [pytest] -DJANGO_SETTINGS_MODULE = backend.settings +DJANGO_SETTINGS_MODULE = config.django.testing addopts = -rP