chore(api): Merge API repository

This commit is contained in:
Pedro De Castro committed 2024-11-25 13:12:54 +01:00
commit 6fe8c81312
102 files changed
+30515

No files matched your search

View File
Whitespace-only changes.
+16
View File
@@ -0,0 +1,16 @@
"""
ASGI config for backend project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
application = get_asgi_application()
+46
View File
@@ -0,0 +1,46 @@
from celery import Celery, Task
celery_app = Celery("tasks")
celery_app.config_from_object("django.conf:settings", namespace="CELERY")
celery_app.conf.update(result_extended=True)
celery_app.autodiscover_tasks(["api"])
class RLSTask(Task):
def apply_async(
self,
args=None,
kwargs=None,
task_id=None,
producer=None,
link=None,
link_error=None,
shadow=None,
**options,
):
from api.models import Task as APITask
from django_celery_results.models import TaskResult
result = super().apply_async(
args=args,
kwargs=kwargs,
task_id=task_id,
producer=producer,
link=link,
link_error=link_error,
shadow=shadow,
**options,
)
task_result_instance = TaskResult.objects.get(task_id=result.task_id)
from api.db_utils import tenant_transaction
tenant_id = kwargs.get("tenant_id")
with tenant_transaction(tenant_id):
APITask.objects.create(
id=task_result_instance.task_id,
tenant_id=tenant_id,
task_runner_task=task_result_instance,
)
return result
+230
View File
@@ -0,0 +1,230 @@
import json
import logging
from enum import StrEnum
from django_guid.log_filters import CorrelationId
from config.env import env
class BackendLogger(StrEnum):
GUNICORN = "gunicorn"
GUNICORN_ACCESS = "gunicorn.access"
GUNICORN_ERROR = "gunicorn.error"
DJANGO = "django"
SECURITY = "django.security"
DB = "django.db"
API = "api"
TASKS = "tasks"
# Formatters
class NDJSONFormatter(logging.Formatter):
"""NDJSON custom formatter for logging messages.
If available, it will include all kind of API request metadata.
"""
def format(self, record):
log_record = {
"timestamp": self.formatTime(record, self.datefmt),
"level": record.levelname,
"message": record.getMessage(),
"logger": record.name,
"module": record.module,
"pathname": record.pathname,
"lineno": record.lineno,
"funcName": record.funcName,
"process": record.process,
"thread": record.thread,
"transaction_id": record.transaction_id
if hasattr(record, "transaction_id")
else None,
}
# Add REST API extra fields
if hasattr(record, "user_id"):
log_record["user_id"] = record.user_id
if hasattr(record, "tenant_id"):
log_record["tenant_id"] = record.tenant_id
if hasattr(record, "method"):
log_record["method"] = record.method
if hasattr(record, "path"):
log_record["path"] = record.path
if hasattr(record, "query_params"):
log_record["query_params"] = record.query_params
if hasattr(record, "duration"):
log_record["duration"] = record.duration
if hasattr(record, "status_code"):
log_record["status_code"] = record.status_code
if record.exc_info:
log_record["exc_info"] = self.formatException(record.exc_info)
return json.dumps(log_record)
class HumanReadableFormatter(logging.Formatter):
"""Human-readable custom formatter for logging messages.
If available, it will include all kinds of API request metadata.
"""
def format(self, record):
log_components = [
f"{self.formatTime(record, self.datefmt)}",
f"[{record.name}]",
f"{record.levelname}:",
f"({record.module})",
f"[module={record.module}",
f"path={record.pathname}",
f"line={record.lineno}",
f"function={record.funcName}",
f"process={record.process}",
f"thread={record.thread}",
f"transaction-id={record.transaction_id if hasattr(record, 'transaction_id') else None}]",
f"{record.getMessage()}",
]
# Add REST API extra fields
if hasattr(record, "user_id"):
log_components.append(f"({record.user_id})")
if hasattr(record, "tenant_id"):
log_components.append(f"[{record.tenant_id}]")
if hasattr(record, "method"):
log_components.append(f'"{record.method} {record.path}"')
if hasattr(record, "query_params"):
log_components.append(f"with parameters {record.query_params}")
if hasattr(record, "duration"):
log_components.append(f"done in {record.duration}s:")
if hasattr(record, "status_code"):
log_components.append(f"{record.status_code}")
if record.exc_info:
log_components.append(self.formatException(record.exc_info))
return " ".join(log_components)
# Filters
class TransactionIdFilter(CorrelationId):
"""Logging filter class.
Used to override the `correlation_id_field` parameter in the parent class to use a different name.
"""
CORRELATION_ID_FIELD = "transaction_id"
def __init__(self):
super().__init__(correlation_id_field=self.CORRELATION_ID_FIELD)
# Logging settings
LEVEL = env("DJANGO_LOGGING_LEVEL", default="INFO")
FORMATTER = env("DJANGO_LOGGING_FORMATTER", default="ndjson")
LOGGING = {
"version": 1,
"disable_existing_loggers": True,
"filters": {"transaction_id": {"()": TransactionIdFilter}},
"formatters": {
"ndjson": {
"()": NDJSONFormatter,
"datefmt": "%Y-%m-%d %H:%M:%S",
},
"human_readable": {
"()": HumanReadableFormatter,
"datefmt": "%Y-%m-%d %H:%M:%S",
},
},
"handlers": {
"gunicorn_console": {
"level": LEVEL,
"class": "logging.StreamHandler",
"formatter": FORMATTER,
"filters": ["transaction_id"],
},
"django_console": {
"level": LEVEL,
"class": "logging.StreamHandler",
"formatter": FORMATTER,
"filters": ["transaction_id"],
},
"api_console": {
"level": LEVEL,
"class": "logging.StreamHandler",
"formatter": FORMATTER,
"filters": ["transaction_id"],
},
"db_console": {
"level": f"{'DEBUG' if LEVEL == 'DEBUG' else 'INFO'}",
"class": "logging.StreamHandler",
"formatter": FORMATTER,
"filters": ["transaction_id"],
},
"security_console": {
"level": LEVEL,
"class": "logging.StreamHandler",
"formatter": FORMATTER,
"filters": ["transaction_id"],
},
"tasks_console": {
"level": LEVEL,
"class": "logging.StreamHandler",
"formatter": FORMATTER,
"filters": ["transaction_id"],
},
},
"loggers": {
BackendLogger.GUNICORN: {
"handlers": ["gunicorn_console"],
"level": LEVEL,
"propagate": False,
},
BackendLogger.GUNICORN_ACCESS: {
"handlers": ["gunicorn_console"],
"level": "CRITICAL",
"propagate": False,
},
BackendLogger.GUNICORN_ERROR: {
"handlers": ["gunicorn_console"],
"level": LEVEL,
"propagate": False,
},
BackendLogger.DJANGO: {
"handlers": ["django_console"],
"level": "WARNING",
"propagate": True,
},
BackendLogger.DB: {
"handlers": ["db_console"],
"level": LEVEL,
"propagate": False,
},
BackendLogger.SECURITY: {
"handlers": ["security_console"],
"level": LEVEL,
"propagate": False,
},
BackendLogger.API: {
"handlers": ["api_console"],
"level": LEVEL,
"propagate": False,
},
BackendLogger.TASKS: {
"handlers": ["tasks_console"],
"level": LEVEL,
"propagate": False,
},
},
# Gunicorn required configuration
"root": {
"level": "ERROR",
"handlers": ["gunicorn_console"],
},
}
Whitespace-only changes.
+209
View File
@@ -0,0 +1,209 @@
from datetime import timedelta
from config.custom_logging import LOGGING # noqa
from config.env import BASE_DIR, env # noqa
from config.settings.celery import * # noqa
from config.settings.partitions import * # noqa
SECRET_KEY = env("SECRET_KEY", default="secret")
DEBUG = env.bool("DJANGO_DEBUG", default=False)
ALLOWED_HOSTS = ["localhost", "127.0.0.1"]
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django.contrib.postgres",
"psqlextra",
"api",
"rest_framework",
"corsheaders",
"drf_spectacular",
"django_guid",
"rest_framework_json_api",
"django_celery_results",
"django_celery_beat",
"rest_framework_simplejwt.token_blacklist",
]
MIDDLEWARE = [
"django_guid.middleware.guid_middleware",
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
"api.middleware.APILoggingMiddleware",
]
CORS_ALLOWED_ORIGINS = ["http://localhost", "http://127.0.0.1"]
ROOT_URLCONF = "config.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
REST_FRAMEWORK = {
"DEFAULT_SCHEMA_CLASS": "drf_spectacular_jsonapi.schemas.openapi.JsonApiAutoSchema",
"DEFAULT_AUTHENTICATION_CLASSES": (
"rest_framework_simplejwt.authentication.JWTAuthentication",
),
"PAGE_SIZE": 10,
"EXCEPTION_HANDLER": "api.exceptions.custom_exception_handler",
"DEFAULT_PAGINATION_CLASS": "drf_spectacular_jsonapi.schemas.pagination.JsonApiPageNumberPagination",
"DEFAULT_PARSER_CLASSES": (
"rest_framework_json_api.parsers.JSONParser",
"rest_framework.parsers.FormParser",
"rest_framework.parsers.MultiPartParser",
),
"DEFAULT_RENDERER_CLASSES": ("api.renderers.APIJSONRenderer",),
"DEFAULT_METADATA_CLASS": "rest_framework_json_api.metadata.JSONAPIMetadata",
"DEFAULT_FILTER_BACKENDS": (
"rest_framework_json_api.filters.QueryParameterValidationFilter",
"rest_framework_json_api.filters.OrderingFilter",
"rest_framework_json_api.django_filters.backends.DjangoFilterBackend",
"rest_framework.filters.SearchFilter",
),
"SEARCH_PARAM": "filter[search]",
"TEST_REQUEST_RENDERER_CLASSES": (
"rest_framework_json_api.renderers.JSONRenderer",
),
"TEST_REQUEST_DEFAULT_FORMAT": "vnd.api+json",
"JSON_API_UNIFORM_EXCEPTIONS": True,
}
SPECTACULAR_SETTINGS = {
"SERVE_INCLUDE_SCHEMA": False,
"COMPONENT_SPLIT_REQUEST": True,
"PREPROCESSING_HOOKS": [
"drf_spectacular_jsonapi.hooks.fix_nested_path_parameters",
],
}
WSGI_APPLICATION = "config.wsgi.application"
DJANGO_GUID = {
"GUID_HEADER_NAME": "Transaction-ID",
"VALIDATE_GUID": True,
"RETURN_HEADER": True,
"EXPOSE_HEADER": True,
"INTEGRATIONS": [],
"IGNORE_URLS": [],
"UUID_LENGTH": 32,
}
DATABASE_ROUTERS = ["api.db_router.MainRouter"]
# Password validation
# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators
AUTH_USER_MODEL = "api.User"
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
"OPTIONS": {"min_length": 12},
},
{
"NAME": "api.validators.MaximumLengthValidator",
"OPTIONS": {
"max_length": 72,
},
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
SIMPLE_JWT = {
# Token lifetime settings
"ACCESS_TOKEN_LIFETIME": timedelta(
minutes=env.int("DJANGO_ACCESS_TOKEN_LIFETIME", 30)
),
"REFRESH_TOKEN_LIFETIME": timedelta(
minutes=env.int("DJANGO_REFRESH_TOKEN_LIFETIME", 60 * 24)
),
"ROTATE_REFRESH_TOKENS": True,
"BLACKLIST_AFTER_ROTATION": True,
# Algorithm and keys
"ALGORITHM": "RS256",
"SIGNING_KEY": env.str("DJANGO_TOKEN_SIGNING_KEY", "").replace("\\n", "\n"),
"VERIFYING_KEY": env.str("DJANGO_TOKEN_VERIFYING_KEY", "").replace("\\n", "\n"),
# Authorization header configuration
"AUTH_HEADER_TYPES": ("Bearer",),
"AUTH_HEADER_NAME": "HTTP_AUTHORIZATION",
# Custom serializers
"TOKEN_OBTAIN_SERIALIZER": "api.serializers.TokenSerializer",
"TOKEN_REFRESH_SERIALIZER": "api.serializers.TokenRefreshSerializer",
# Standard JWT claims
"TOKEN_TYPE_CLAIM": "typ",
"JTI_CLAIM": "jti",
"USER_ID_FIELD": "id",
"USER_ID_CLAIM": "sub",
# Issuer and Audience claims, for the moment we will keep these values as default values, they may change in the future.
"AUDIENCE": env.str("DJANGO_JWT_AUDIENCE", "https://api.prowler.com"),
"ISSUER": env.str("DJANGO_JWT_ISSUER", "https://api.prowler.com"),
# Additional security settings
"UPDATE_LAST_LOGIN": True,
}
SECRETS_ENCRYPTION_KEY = env.str("DJANGO_SECRETS_ENCRYPTION_KEY", "")
# Internationalization
# https://docs.djangoproject.com/en/5.0/topics/i18n/
LANGUAGE_CODE = "en-us"
LANGUAGES = [
("en", "English"),
]
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/5.0/howto/static-files/
STATIC_URL = "static/"
# Default primary key field type
# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
# Cache settings
CACHE_MAX_AGE = env.int("DJANGO_CACHE_MAX_AGE", 3600)
CACHE_STALE_WHILE_REVALIDATE = env.int("DJANGO_STALE_WHILE_REVALIDATE", 60)
TESTING = False
+40
View File
@@ -0,0 +1,40 @@
from config.django.base import * # noqa
from config.env import env
DEBUG = env.bool("DJANGO_DEBUG", default=True)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"])
# Database
DATABASES = {
"prowler_user": {
"ENGINE": "psqlextra.backend",
"NAME": env("POSTGRES_DB", default="prowler_db"),
"USER": env("POSTGRES_USER", default="prowler_user"),
"PASSWORD": env("POSTGRES_PASSWORD", default="prowler"),
"HOST": env("POSTGRES_HOST", default="postgres-db"),
"PORT": env("POSTGRES_PORT", default="5432"),
},
"admin": {
"ENGINE": "psqlextra.backend",
"NAME": env("POSTGRES_DB", default="prowler_db"),
"USER": env("POSTGRES_ADMIN_USER", default="prowler"),
"PASSWORD": env("POSTGRES_ADMIN_PASSWORD", default="S3cret"),
"HOST": env("POSTGRES_HOST", default="postgres-db"),
"PORT": env("POSTGRES_PORT", default="5432"),
},
}
DATABASES["default"] = DATABASES["prowler_user"]
REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = tuple( # noqa: F405
render_class
for render_class in REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] # noqa: F405
) + ("rest_framework_json_api.renderers.BrowsableAPIRenderer",)
REST_FRAMEWORK["DEFAULT_FILTER_BACKENDS"] = tuple( # noqa: F405
filter_backend
for filter_backend in REST_FRAMEWORK["DEFAULT_FILTER_BACKENDS"] # noqa: F405
if "DjangoFilterBackend" not in filter_backend
) + ("api.filters.CustomDjangoFilterBackend",)
SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0="
@@ -0,0 +1,28 @@
from config.django.base import * # noqa
from config.env import env
DEBUG = env.bool("DJANGO_DEBUG", default=False)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
# Database
# TODO Use Django database routers https://docs.djangoproject.com/en/5.0/topics/db/multi-db/#automatic-database-routing
DATABASES = {
"prowler_user": {
"ENGINE": "django.db.backends.postgresql",
"NAME": env("POSTGRES_DB"),
"USER": env("POSTGRES_USER"),
"PASSWORD": env("POSTGRES_PASSWORD"),
"HOST": env("POSTGRES_HOST"),
"PORT": env("POSTGRES_PORT"),
},
"admin": {
"ENGINE": "psqlextra.backend",
"NAME": env("POSTGRES_DB"),
"USER": env("POSTGRES_ADMIN_USER"),
"PASSWORD": env("POSTGRES_ADMIN_PASSWORD"),
"HOST": env("POSTGRES_HOST"),
"PORT": env("POSTGRES_PORT"),
},
}
DATABASES["default"] = DATABASES["prowler_user"]
+26
View File
@@ -0,0 +1,26 @@
from config.django.base import * # noqa
from config.env import env
DEBUG = env.bool("DJANGO_DEBUG", default=False)
ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["localhost", "127.0.0.1"])
DATABASES = {
"default": {
"ENGINE": "psqlextra.backend",
"NAME": "prowler_db_test",
"USER": env("POSTGRES_USER", default="prowler"),
"PASSWORD": env("POSTGRES_PASSWORD", default="S3cret"),
"HOST": env("POSTGRES_HOST", default="localhost"),
"PORT": env("POSTGRES_PORT", default="5432"),
},
}
DATABASE_ROUTERS = []
TESTING = True
SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0="
# JWT
SIMPLE_JWT["ALGORITHM"] = "HS256" # noqa: F405
+7
View File
@@ -0,0 +1,7 @@
from pathlib import Path
import environ
env = environ.Env()
BASE_DIR = Path(__file__).resolve().parent.parent
+43
View File
@@ -0,0 +1,43 @@
import logging
import multiprocessing
import os
from config.env import env
# Ensure the environment variable for Django settings is set
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
# Import Django and set it up before accessing settings
import django # noqa: E402
django.setup()
from config.django.production import LOGGING as DJANGO_LOGGERS, DEBUG # noqa: E402
from config.custom_logging import BackendLogger # noqa: E402
BIND_ADDRESS = env("DJANGO_BIND_ADDRESS", default="127.0.0.1")
PORT = env("DJANGO_PORT", default=8000)
# Server settings
bind = f"{BIND_ADDRESS}:{PORT}"
workers = env.int("DJANGO_WORKERS", default=multiprocessing.cpu_count() * 2 + 1)
reload = DEBUG
# Logging
logconfig_dict = DJANGO_LOGGERS
gunicorn_logger = logging.getLogger(BackendLogger.GUNICORN)
# Hooks
def on_starting(_):
gunicorn_logger.info(f"Starting gunicorn server with {workers} workers")
if reload:
gunicorn_logger.warning("Reload settings enabled (dev mode)")
def on_reload(_):
gunicorn_logger.warning("Gunicorn server has reloaded")
def when_ready(_):
gunicorn_logger.info("Gunicorn server is ready")
@@ -0,0 +1,3 @@
from config.celery import celery_app
__all__ = ("celery_app",)
+11
View File
@@ -0,0 +1,11 @@
from config.env import env
VALKEY_HOST = env("VALKEY_HOST", default="valkey")
VALKEY_PORT = env("VALKEY_PORT", default="6379")
VALKEY_DB = env("VALKEY_DB", default="0")
CELERY_BROKER_URL = f"redis://{VALKEY_HOST}:{VALKEY_PORT}/{VALKEY_DB}"
CELERY_RESULT_BACKEND = "django-db"
CELERY_TASK_TRACK_STARTED = True
CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
@@ -0,0 +1,16 @@
from config.env import env
# Partitioning
PSQLEXTRA_PARTITIONING_MANAGER = "api.partitions.manager"
# Set the months for each partition. Setting the partition months to 1 will create partitions with a size of 1 natural month.
FINDINGS_TABLE_PARTITION_MONTHS = env.int("FINDINGS_TABLE_PARTITION_MONTHS", 1)
# Set the number of partitions to create
FINDINGS_TABLE_PARTITION_COUNT = env.int("FINDINGS_TABLE_PARTITION_COUNT", 7)
# Set the number of months to keep partitions before deleting them
# Setting this to None will keep partitions indefinitely
FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS = env.int(
"FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS", None
)
+7
View File
@@ -0,0 +1,7 @@
from django.contrib import admin
from django.urls import include, path
urlpatterns = [
path("admin/", admin.site.urls),
path("api/v1/", include("api.v1.urls")),
]
+16
View File
@@ -0,0 +1,16 @@
"""
WSGI config for backend project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django.production")
application = get_wsgi_application()