diff --git a/.env b/.env index f405293de7..71a63171e7 100644 --- a/.env +++ b/.env @@ -78,6 +78,9 @@ TASK_RETRY_ATTEMPTS=5 # Valkey settings # If running Valkey and celery on host, use localhost, else use 'valkey' +VALKEY_SCHEME=redis +VALKEY_USERNAME= +VALKEY_PASSWORD= VALKEY_HOST=valkey VALKEY_PORT=6379 VALKEY_DB=0 diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index abaf3dbbe8..e27c1894d2 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -22,6 +22,9 @@ env: POSTGRES_USER: prowler_user POSTGRES_PASSWORD: prowler POSTGRES_DB: postgres-db + VALKEY_SCHEME: redis + VALKEY_USERNAME: "" + VALKEY_PASSWORD: "" VALKEY_HOST: localhost VALKEY_PORT: 6379 VALKEY_DB: 0 diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 8141929052..c75f3ce40b 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -4,9 +4,9 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.24.0] (Prowler UNRELEASED) -### 🔐 Security +### 🚀 Added -- Pin all unpinned dependencies to exact versions to prevent supply chain attacks and ensure reproducible builds [(#10469)](https://github.com/prowler-cloud/prowler/pull/10469) +- `VALKEY_SCHEME`, `VALKEY_USERNAME`, and `VALKEY_PASSWORD` environment variables to configure Celery broker TLS/auth connection details for Valkey/ElastiCache [(#10420)](https://github.com/prowler-cloud/prowler/pull/10420) ### 🐞 Fixed @@ -18,6 +18,12 @@ All notable changes to the **Prowler API** are documented in this file. - Finding groups muted filter, counters, metadata extraction and mute reaggregation [(#10477)](https://github.com/prowler-cloud/prowler/pull/10477) - Finding groups `check_title__icontains` resolution, `name__icontains` resource filter and `resource_group` field in `/resources` response [(#10486)](https://github.com/prowler-cloud/prowler/pull/10486) +### 🔐 Security + +- Pin all unpinned dependencies to exact versions to prevent supply chain attacks and ensure reproducible builds [(#10469)](https://github.com/prowler-cloud/prowler/pull/10469) + +--- + ## [1.23.0] (Prowler v5.22.0) ### 🚀 Added diff --git a/api/src/backend/api/tests/test_celery_settings.py b/api/src/backend/api/tests/test_celery_settings.py new file mode 100644 index 0000000000..d4010796ee --- /dev/null +++ b/api/src/backend/api/tests/test_celery_settings.py @@ -0,0 +1,43 @@ +import pytest +from config.settings.celery import _build_celery_broker_url + + +class TestBuildCeleryBrokerUrl: + def test_without_credentials(self): + broker_url = _build_celery_broker_url("redis", "", "", "valkey", "6379", "0") + + assert broker_url == "redis://valkey:6379/0" + + def test_with_password_only(self): + broker_url = _build_celery_broker_url( + "rediss", "", "secret", "cache.example.com", "6379", "0" + ) + + assert broker_url == "rediss://:secret@cache.example.com:6379/0" + + def test_with_username_and_password(self): + broker_url = _build_celery_broker_url( + "rediss", "default", "secret", "cache.example.com", "6379", "0" + ) + + assert broker_url == "rediss://default:secret@cache.example.com:6379/0" + + def test_with_username_only(self): + broker_url = _build_celery_broker_url( + "redis", "admin", "", "valkey", "6379", "0" + ) + + assert broker_url == "redis://admin@valkey:6379/0" + + def test_url_encodes_credentials(self): + broker_url = _build_celery_broker_url( + "rediss", "user@name", "p@ss:word", "cache.example.com", "6379", "0" + ) + + assert ( + broker_url == "rediss://user%40name:p%40ss%3Aword@cache.example.com:6379/0" + ) + + def test_invalid_scheme_raises_error(self): + with pytest.raises(ValueError, match="Invalid VALKEY_SCHEME 'http'"): + _build_celery_broker_url("http", "", "", "valkey", "6379", "0") diff --git a/api/src/backend/config/settings/celery.py b/api/src/backend/config/settings/celery.py index a2aba00007..b7030ebea4 100644 --- a/api/src/backend/config/settings/celery.py +++ b/api/src/backend/config/settings/celery.py @@ -1,10 +1,52 @@ +from urllib.parse import quote + from config.env import env +_VALID_SCHEMES = {"redis", "rediss"} + + +def _build_celery_broker_url( + scheme: str, + username: str, + password: str, + host: str, + port: str, + db: str, +) -> str: + if scheme not in _VALID_SCHEMES: + raise ValueError( + f"Invalid VALKEY_SCHEME '{scheme}'. Must be one of: {', '.join(sorted(_VALID_SCHEMES))}" + ) + + encoded_username = quote(username, safe="") if username else "" + encoded_password = quote(password, safe="") if password else "" + + auth = "" + if encoded_username and encoded_password: + auth = f"{encoded_username}:{encoded_password}@" + elif encoded_password: + auth = f":{encoded_password}@" + elif encoded_username: + auth = f"{encoded_username}@" + + return f"{scheme}://{auth}{host}:{port}/{db}" + + +VALKEY_SCHEME = env("VALKEY_SCHEME", default="redis") +VALKEY_USERNAME = env("VALKEY_USERNAME", default="") +VALKEY_PASSWORD = env("VALKEY_PASSWORD", default="") VALKEY_HOST = env("VALKEY_HOST", default="valkey") VALKEY_PORT = env("VALKEY_PORT", default="6379") VALKEY_DB = env("VALKEY_DB", default="0") -CELERY_BROKER_URL = f"redis://{VALKEY_HOST}:{VALKEY_PORT}/{VALKEY_DB}" +CELERY_BROKER_URL = _build_celery_broker_url( + VALKEY_SCHEME, + VALKEY_USERNAME, + VALKEY_PASSWORD, + VALKEY_HOST, + VALKEY_PORT, + VALKEY_DB, +) CELERY_RESULT_BACKEND = "django-db" CELERY_TASK_TRACK_STARTED = True diff --git a/contrib/k8s/helm/prowler-api/values.yaml b/contrib/k8s/helm/prowler-api/values.yaml index 40a7fda298..61146e1e1f 100644 --- a/contrib/k8s/helm/prowler-api/values.yaml +++ b/contrib/k8s/helm/prowler-api/values.yaml @@ -39,6 +39,9 @@ secrets: POSTGRES_PASSWORD: POSTGRES_DB: # Valkey settings + VALKEY_SCHEME: redis + VALKEY_USERNAME: + VALKEY_PASSWORD: VALKEY_HOST: valkey-headless VALKEY_PORT: "6379" VALKEY_DB: "0" diff --git a/contrib/k8s/helm/prowler-app/templates/api/secret-valkey.yaml b/contrib/k8s/helm/prowler-app/templates/api/secret-valkey.yaml index 7778d06731..9b84dd33d6 100644 --- a/contrib/k8s/helm/prowler-app/templates/api/secret-valkey.yaml +++ b/contrib/k8s/helm/prowler-app/templates/api/secret-valkey.yaml @@ -7,6 +7,9 @@ metadata: {{- include "prowler.labels" . | nindent 4 }} type: Opaque stringData: + VALKEY_SCHEME: {{ .Values.valkey.scheme | default "redis" | quote }} + VALKEY_USERNAME: {{ .Values.valkey.username | default "" | quote }} + VALKEY_PASSWORD: {{ .Values.valkey.password | default "" | quote }} VALKEY_HOST: "{{ include "prowler.fullname" . }}-valkey" VALKEY_PORT: "6379" VALKEY_DB: "0" diff --git a/contrib/k8s/helm/prowler-app/values.yaml b/contrib/k8s/helm/prowler-app/values.yaml index f855210262..9a162bd67e 100644 --- a/contrib/k8s/helm/prowler-app/values.yaml +++ b/contrib/k8s/helm/prowler-app/values.yaml @@ -529,6 +529,9 @@ postgresql: valkey: # If enabled, it will create a Secret with the following. # Otherwise, create a secret with + # - VALKEY_SCHEME + # - VALKEY_USERNAME + # - VALKEY_PASSWORD # - VALKEY_HOST # - VALKEY_PORT # - VALKEY_DB diff --git a/skills/prowler-api/assets/celery_patterns.py b/skills/prowler-api/assets/celery_patterns.py index 700e63891b..ee1b09bfa6 100644 --- a/skills/prowler-api/assets/celery_patterns.py +++ b/skills/prowler-api/assets/celery_patterns.py @@ -249,7 +249,12 @@ celery_app.conf.update( # Django settings (config/settings/celery.py) """ -CELERY_BROKER_URL = f"redis://{VALKEY_HOST}:{VALKEY_PORT}/{VALKEY_DB}" +VALKEY_SCHEME = env("VALKEY_SCHEME", default="redis") +VALKEY_USERNAME = env("VALKEY_USERNAME", default="") +VALKEY_PASSWORD = env("VALKEY_PASSWORD", default="") +CELERY_BROKER_URL = _build_celery_broker_url( + VALKEY_SCHEME, VALKEY_USERNAME, VALKEY_PASSWORD, VALKEY_HOST, VALKEY_PORT, VALKEY_DB +) CELERY_RESULT_BACKEND = "django-db" # Store results in PostgreSQL CELERY_TASK_TRACK_STARTED = True # Track when tasks start CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True diff --git a/skills/prowler-api/references/configuration.md b/skills/prowler-api/references/configuration.md index dac5c735bf..677f999cc4 100644 --- a/skills/prowler-api/references/configuration.md +++ b/skills/prowler-api/references/configuration.md @@ -169,11 +169,21 @@ DATABASES = { ### Broker/Backend ```python +VALKEY_SCHEME = env("VALKEY_SCHEME", default="redis") +VALKEY_USERNAME = env("VALKEY_USERNAME", default="") +VALKEY_PASSWORD = env("VALKEY_PASSWORD", default="") VALKEY_HOST = env("VALKEY_HOST", default="valkey") VALKEY_PORT = env("VALKEY_PORT", default="6379") VALKEY_DB = env("VALKEY_DB", default="0") -CELERY_BROKER_URL = f"redis://{VALKEY_HOST}:{VALKEY_PORT}/{VALKEY_DB}" +CELERY_BROKER_URL = _build_celery_broker_url( + VALKEY_SCHEME, + VALKEY_USERNAME, + VALKEY_PASSWORD, + VALKEY_HOST, + VALKEY_PORT, + VALKEY_DB, +) CELERY_RESULT_BACKEND = "django-db" # Store results in PostgreSQL CELERY_TASK_TRACK_STARTED = True CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True diff --git a/skills/prowler-api/references/production-settings.md b/skills/prowler-api/references/production-settings.md index ea9b59c328..12b64483bf 100644 --- a/skills/prowler-api/references/production-settings.md +++ b/skills/prowler-api/references/production-settings.md @@ -164,8 +164,13 @@ POSTGRES_DB=... POSTGRES_USER=... POSTGRES_PASSWORD=... -# Redis (for Celery) -REDIS_URL=redis://host:6379/0 +# Valkey/Redis (for Celery) +VALKEY_SCHEME=rediss +VALKEY_USERNAME=default +VALKEY_PASSWORD= +VALKEY_HOST=host +VALKEY_PORT=6379 +VALKEY_DB=0 # Optional SENTRY_DSN=https://...@sentry.io/...