diff --git a/.env.example b/.env.example index 8698b5c225..f704c0086f 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,7 @@ DJANGO_ACCESS_TOKEN_LIFETIME=30 DJANGO_REFRESH_TOKEN_LIFETIME=1440 DJANGO_CACHE_MAX_AGE=3600 DJANGO_STALE_WHILE_REVALIDATE=60 +DJANGO_SECRETS_ENCRYPTION_KEY="" # PostgreSQL settings # If running django and celery on host, use 'localhost', else use 'postgres-db' diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 2c7cadf01c..527f227df7 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -28,7 +28,7 @@ start_prod_server() { start_worker() { echo "Starting the worker..." - poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q default,scans -E + poetry run python -m celery -A config.celery worker -l "${DJANGO_LOGGING_LEVEL:-info}" -Q celery,scans -E } case "$1" in diff --git a/poetry.lock b/poetry.lock index 342722a07a..ef3f1dfdc9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1297,15 +1297,13 @@ wrapt = ">=1.10,<2" dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] -name = "detect-secrets" +name = "detect_secrets" version = "1.5.0" description = "Tool for detecting secrets in the codebase" optional = false python-versions = "*" -files = [ - {file = "detect_secrets-1.5.0-py3-none-any.whl", hash = "sha256:e24e7b9b5a35048c313e983f76c4bd09dad89f045ff059e354f9943bf45aa060"}, - {file = "detect_secrets-1.5.0.tar.gz", hash = "sha256:6bb46dcc553c10df51475641bb30fd69d25645cc12339e46c824c1e0c388898a"}, -] +files = [] +develop = false [package.dependencies] pyyaml = "*" @@ -1315,6 +1313,12 @@ requests = "*" gibberish = ["gibberish-detector"] word-list = ["pyahocorasick"] +[package.source] +type = "git" +url = "https://github.com/Yelp/detect-secrets.git" +reference = "master" +resolved_reference = "462720710ec337300fab2b4f2290949c7ee141eb" + [[package]] name = "dill" version = "0.3.9" @@ -3258,7 +3262,7 @@ colorama = "0.4.6" cryptography = "43.0.1" dash = "2.18.1" dash-bootstrap-components = "1.6.0" -detect-secrets = "1.5.0" +detect-secrets = {git = "https://github.com/Yelp/detect-secrets.git", rev = "master"} google-api-python-client = "2.147.0" google-auth-httplib2 = ">=0.1,<0.3" jsonschema = "4.23.0" @@ -3281,7 +3285,7 @@ tzlocal = "5.2" type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "master" -resolved_reference = "30bc971f4b289cf4c53ddd10fa7cfba425c32a98" +resolved_reference = "03a26ec507b2b93b6787be4b0516950d6ab3c092" [[package]] name = "psutil" diff --git a/src/backend/api/db_utils.py b/src/backend/api/db_utils.py index b597a34676..186932c5a0 100644 --- a/src/backend/api/db_utils.py +++ b/src/backend/api/db_utils.py @@ -225,3 +225,15 @@ class StatusEnum(EnumType): class StatusEnumField(PostgresEnumField): def __init__(self, *args, **kwargs): super().__init__("status", *args, **kwargs) + + +# Postgres enum definition for Provider secrets type + + +class ProviderSecretTypeEnum(EnumType): + enum_type_name = "provider_secret_type" + + +class ProviderSecretTypeEnumField(PostgresEnumField): + def __init__(self, *args, **kwargs): + super().__init__("provider_secret_type", *args, **kwargs) diff --git a/src/backend/api/filters.py b/src/backend/api/filters.py index 341ef0e196..257cf368fa 100644 --- a/src/backend/api/filters.py +++ b/src/backend/api/filters.py @@ -32,6 +32,7 @@ from api.models import ( Finding, SeverityChoices, StatusChoices, + ProviderSecret, ) from api.rls import Tenant from api.uuid_utils import ( @@ -385,3 +386,15 @@ class FindingFilter(FilterSet): if isinstance(value, date): dt = datetime.combine(value, datetime.min.time(), tzinfo=timezone.utc) return dt + + +class ProviderSecretFilter(FilterSet): + inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") + updated_at = DateFilter(field_name="updated_at", lookup_expr="date") + provider = UUIDFilter(field_name="provider__id", lookup_expr="exact") + + class Meta: + model = ProviderSecret + fields = { + "name": ["exact", "icontains"], + } diff --git a/src/backend/api/fixtures/2_dev_providers.json b/src/backend/api/fixtures/2_dev_providers.json index 22292f199b..afd5464803 100644 --- a/src/backend/api/fixtures/2_dev_providers.json +++ b/src/backend/api/fixtures/2_dev_providers.json @@ -105,5 +105,44 @@ "metadata": {}, "scanner_args": {} } + }, + { + "model": "api.providersecret", + "pk": "11491b47-75ae-4f71-ad8d-3e630a72182e", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "inserted_at": "2024-10-11T08:03:05.026Z", + "updated_at": "2024-10-11T08:04:47.033Z", + "name": "GCP static secrets", + "secret_type": "static", + "_secret": "Z0FBQUFBQm5DTndmZW9KakRZUHM2UHhQN2V3RzN0QmM1cERham8yMHp5cnVTT0lzdGFyS1FuVmJXUlpYSGsyU0cxR3RMMEdQYXlYMUVsaWtqLU1OZWlaVUp6OFREYlotZTVBY3BuTlZYbm9YcUJydzAxV2p5dkpLamI1Y2tUYzA0MmJUNWxsNTBRM0E1SDRCa0pPQWVlb05YU3dfeUhkLTRmOEh3dGczOGh1ZGhQcVdZdVAtYmtoSWlwNXM4VGFoVmF3dno2X1hrbk5GZjZTWjVuWEdEZUFXeHJSQjEzbTlVakhNdzYyWTdiVEpvUEc2MTNpRzUtczhEank1eGI0b3MyMlAyaGN6dlByZmtUWHByaDNUYWFqYS1tYnNBUkRKTzBacFNSRjFuVmd5bUtFUEJhd1ZVS1ZDd2xSUV9PaEtLTnc0XzVkY2lhM01WTjQwaWdJSk9wNUJSXzQ4RUNQLXFPNy1VdzdPYkZyWkVkU3RyQjVLTS1MVHN0R3k4THNKZ2NBNExaZnl3Q1EwN2dwNGRsUXptMjB0LXUzTUpzTDE2Q1hmS0ZSN2g1ZjBPeV8taFoxNUwxc2FEcktXX0dCM1IzeUZTTHNiTmNxVXBvNWViZTJScUVWV2VYTFQ4UHlid21PY1A0UjdNMGtERkZCd0lLMlJENDMzMVZUM09DQ0twd1N3VHlZd09XLUctOWhYcFJIR1p5aUlZeEUzejc2dWRYdGNsd0xOODNqRUFEczhSTWNtWU0tdFZ1ZTExaHNHUVYtd0Zxdld1LTdKVUNINzlZTGdHODhKeVVpQmRZMHRUNTJRRWhwS1F1Y3I2X2Iwc0c1NHlXSVRLZWxreEt0dVRnOTZFMkptU2VMS1dWXzdVOVRzMUNUWXM2aFlxVDJXdGo3d2cxSVZGWlI2ZWhIZzZBcEl4bEJ6UnVHc0RYWVNHcjFZUHI5ZUYyWG9rSlo0QUVSUkFCX3h2UmtJUTFzVXJUZ25vTmk2VzdoTTNta05ucmNfTi0yR1ZxN1E2MnZJOVVKOGxmMXMzdHMxVndmSVhQbUItUHgtMVpVcHJwMU5JVHJLb0Y1aHV5OEEwS0kzQkEtcFJkdkRnWGxmZnprNFhndWg1TmQyd09yTFdTRmZ3d2ZvZFUtWXp4a2VYb3JjckFIcE13MDUzX0RHSnlzM0N2ZE5IRzJzMXFMc0k4MDRyTHdLZFlWOG9SaFF0LU43Ynd6VFlEcVNvdFZ0emJEVk10aEp4dDZFTFNFNzk0UUo2WTlVLWRGYm1fanZHaFZreHBIMmtzVjhyS0xPTk9fWHhiVTJHQXZwVlVuY3JtSjFUYUdHQzhEaHFNZXhwUHBmY0kxaUVrOHo4a0FYOTdpZVJDbFRvdFlQeWo3eFZHX1ZMZ1Myc3prU3o2c3o2eXNja1U4N0Y1T0d1REVjZFRGNTByUkgyemVCSjlQYkY2bmJ4YTZodHB0cUNzd2xZcENycUdsczBIaEZPbG1jVUlqNlM2cEE3aGpVaWswTzBDLVFGUHM5UHhvM09saWNtaDhaNVlsc3FZdktKeWlheDF5OGhTODE2N3JWamdTZG5Fa3JSQ2ZUSEVfRjZOZXdreXRZLTBZRFhleVFFeC1YUzc0cWhYeEhobGxvdnZ3Rm15WFlBWXp0dm1DeTA5eExLeEFRRXVRSXBXdTNEaWdZZ3JDenItdDhoZlFiTzI0SGZ1c01FR1FNaFVweVBKR1YxWGRUMW1Mc2JVdW9raWR6UHk2ZTBnS05pV3oyZVBjREdkY3k4ZHZPUWE5S281MkJRSHF3NnpTclZ5bl90bk1wUEh6Tkp5dXlDcE5paWRqcVhxRFVObWIzRldWOGJ2aC1CRHZpbFZrb0hjNGpCMm5POGRiS2lETUpMLUVfQlhCdTZPLW9USW1LTFlTSF9zRUJYZ1NKeFFEQjNOR215ZXJDbkFndmcxWl9rWlk9", + "provider": "8851db6b-42e5-4533-aa9e-30a32d67e875" + } + }, + { + "model": "api.providersecret", + "pk": "40191ad5-d8c2-40a9-826d-241397626b68", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "inserted_at": "2024-10-10T11:11:44.515Z", + "updated_at": "2024-10-11T07:59:56.102Z", + "name": "AWS static secrets", + "secret_type": "static", + "_secret": "Z0FBQUFBQm5DTnI4Y1RyV19UWEJzc3kzQUExcU5tdlQzbFVLeDdZMWd1MzkwWkl2UF9oZGhiVEJHVWpSMXV4MjYyN3g2OVpvNVpkQUQ3S0VGaGdQLTFhQWE3MkpWZUt2cnVhODc4d3FpY3FVZkpwdHJzNUJPeFRwZ3N4bGpPZTlkNWRNdFlwTHU3aTNWR3JjSzJwLWRITHdfQWpXb1F0c1l3bVFxbnFrTEpPTGgxcnF1VUprSzZ5dGRQU2VGYmZhTTlwbVpsNFBNWlFhVW9RbjJyYnZ5N0oweE5kV0ZEaUdpUUpNVExOa3oyQ2dNREVSenJ0TEFZc0RrRWpXNUhyMmtybGNLWDVOR0FabEl4QVR1bkZyb2hBLWc1MFNIekVyeXI0SmVreHBjRnJ1YUlVdXpVbW9JZkk0aEgxYlM1VGhSRlhtcS14YzdTYUhXR2xodElmWjZuNUVwaHozX1RVTG1QWHdPZWd4clNHYnAyOTBsWEl5UU83RGxZb0RKWjdadjlsTmJtSHQ0Yl9uaDJoODB0QV9sWmFYbFAxcjA1bmhNVlNqc2xEeHlvcUJFbVZvY250ZENnMnZLT1psb1JDclB3WVR6NGdZb2pzb3U4Ny04QlB0UTZub0dMOXZEUTZEcVJhZldCWEZZSDdLTy02UVZqck5zVTZwS3pObGlOejNJeHUzbFRabFM2V2xaekZVRjZtX3VzZlplendnOWQzT01WMFd3ejNadHVlTFlqRGR2dk5Da29zOFYwOUdOaEc4OHhHRnJFMmJFMk12VDNPNlBBTGlsXy13cUM1QkVYb0o1Z2U4ZXJnWXpZdm1sWjA5bzQzb2NFWC1xbmIycGZRbGtCaGNaOWlkX094UUNNampwbkZoREctNWI4QnZRaE8zM3BEQ1BwNzA1a3BzOGczZXdIM2s1NHFGN1ZTbmJhZkc4RVdfM0ZIZU5udTBYajd1RGxpWXZpRWdSMmhHa2RKOEIzbmM0X2F1OGxrN2p6LW9UVldDOFVpREoxZ1UzcTBZX19OQ0xJb0syWlhNSlQ4MzQwdzRtVG94Y01GS3FMLV95UVlxOTFORk8zdjE5VGxVaXdhbGlzeHdoYWNzazZWai1GUGtUM2gzR0ZWTTY4SThWeVFnZldIaklOTTJqTTg1VkhEYW5wNmdEVllXMmJCV2tpVmVYeUV2c0E1T00xbHJRNzgzVG9wb0Q1cV81UEhqYUFsQ2p1a0VpRDVINl9SVkpyZVRNVnVXQUxwY3NWZnJrNmRVREpiLWNHYUpXWmxkQlhNbWhuR1NmQ1BaVDlidUxCWHJMaHhZbk1FclVBaEVZeWg1ZlFoenZzRHlKbV8wa3lmMGZrd3NmTDZjQkE0UXNSUFhpTWtUUHBrX29BVzc4QzEtWEJIQW1GMGFuZVlXQWZIOXJEamloeGFCeHpYMHNjMFVfNXpQdlJfSkk2bzFROU5NU0c1SHREWW1nbkFNZFZ0UjdPRGdjaF96RGplY1hjdFFzLVR6MTVXYlRjbHIxQ2JRejRpVko5NWhBU0ZHR3ZvczU5elljRGpHRTdIc0FsSm5fUHEwT1gtTS1lN3M3X3ZZRnlkYUZoZXRQeEJsZlhLdFdTUzU1NUl4a29aOWZIdTlPM0Fnak1xYWVkYTNiMmZXUHlXS2lwUVBZLXQyaUxuRmtQNFFieE9SVmdZVW9WTHlzbnBPZlNIdGVHOE1LNVNESjN3cGtVSHVpT1NJWHE1ZzNmUTVTOC0xX3NGSmJqU19IbjZfQWtMRG1YNUQtRy13TUJIZFlyOXJkQzFQbkdZVXVzM2czbS1HWHFBT1pXdVd3N09tcG82SVhnY1ZtUWxqTEg2UzJCUmllb2pweVN2aGwwS1FVRUhjNEN2amRMc3MwVU4zN3dVMWM5Slg4SERtenFaQk1yMWx0LWtxVWtLZVVtbU4yejVEM2h6TEt0RGdfWE09", + "provider": "b85601a8-4b45-4194-8135-03fb980ef428" + } + }, + { + "model": "api.providersecret", + "pk": "ed89d1ea-366a-4d12-a602-f2ab77019742", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "inserted_at": "2024-10-10T11:11:44.515Z", + "updated_at": "2024-10-11T07:59:56.102Z", + "name": "Azure static secrets", + "secret_type": "static", + "_secret": "Z0FBQUFBQm5DTnI4Y1RyV19UWEJzc3kzQUExcU5tdlQzbFVLeDdZMWd1MzkwWkl2UF9oZGhiVEJHVWpSMXV4MjYyN3g2OVpvNVpkQUQ3S0VGaGdQLTFhQWE3MkpWZUt2cnVhODc4d3FpY3FVZkpwdHJzNUJPeFRwZ3N4bGpPZTlkNWRNdFlwTHU3aTNWR3JjSzJwLWRITHdfQWpXb1F0c1l3bVFxbnFrTEpPTGgxcnF1VUprSzZ5dGRQU2VGYmZhTTlwbVpsNFBNWlFhVW9RbjJyYnZ5N0oweE5kV0ZEaUdpUUpNVExOa3oyQ2dNREVSenJ0TEFZc0RrRWpXNUhyMmtybGNLWDVOR0FabEl4QVR1bkZyb2hBLWc1MFNIekVyeXI0SmVreHBjRnJ1YUlVdXpVbW9JZkk0aEgxYlM1VGhSRlhtcS14YzdTYUhXR2xodElmWjZuNUVwaHozX1RVTG1QWHdPZWd4clNHYnAyOTBsWEl5UU83RGxZb0RKWjdadjlsTmJtSHQ0Yl9uaDJoODB0QV9sWmFYbFAxcjA1bmhNVlNqc2xEeHlvcUJFbVZvY250ZENnMnZLT1psb1JDclB3WVR6NGdZb2pzb3U4Ny04QlB0UTZub0dMOXZEUTZEcVJhZldCWEZZSDdLTy02UVZqck5zVTZwS3pObGlOejNJeHUzbFRabFM2V2xaekZVRjZtX3VzZlplendnOWQzT01WMFd3ejNadHVlTFlqRGR2dk5Da29zOFYwOUdOaEc4OHhHRnJFMmJFMk12VDNPNlBBTGlsXy13cUM1QkVYb0o1Z2U4ZXJnWXpZdm1sWjA5bzQzb2NFWC1xbmIycGZRbGtCaGNaOWlkX094UUNNampwbkZoREctNWI4QnZRaE8zM3BEQ1BwNzA1a3BzOGczZXdIM2s1NHFGN1ZTbmJhZkc4RVdfM0ZIZU5udTBYajd1RGxpWXZpRWdSMmhHa2RKOEIzbmM0X2F1OGxrN2p6LW9UVldDOFVpREoxZ1UzcTBZX19OQ0xJb0syWlhNSlQ4MzQwdzRtVG94Y01GS3FMLV95UVlxOTFORk8zdjE5VGxVaXdhbGlzeHdoYWNzazZWai1GUGtUM2gzR0ZWTTY4SThWeVFnZldIaklOTTJqTTg1VkhEYW5wNmdEVllXMmJCV2tpVmVYeUV2c0E1T00xbHJRNzgzVG9wb0Q1cV81UEhqYUFsQ2p1a0VpRDVINl9SVkpyZVRNVnVXQUxwY3NWZnJrNmRVREpiLWNHYUpXWmxkQlhNbWhuR1NmQ1BaVDlidUxCWHJMaHhZbk1FclVBaEVZeWg1ZlFoenZzRHlKbV8wa3lmMGZrd3NmTDZjQkE0UXNSUFhpTWtUUHBrX29BVzc4QzEtWEJIQW1GMGFuZVlXQWZIOXJEamloeGFCeHpYMHNjMFVfNXpQdlJfSkk2bzFROU5NU0c1SHREWW1nbkFNZFZ0UjdPRGdjaF96RGplY1hjdFFzLVR6MTVXYlRjbHIxQ2JRejRpVko5NWhBU0ZHR3ZvczU5elljRGpHRTdIc0FsSm5fUHEwT1gtTS1lN3M3X3ZZRnlkYUZoZXRQeEJsZlhLdFdTUzU1NUl4a29aOWZIdTlPM0Fnak1xYWVkYTNiMmZXUHlXS2lwUVBZLXQyaUxuRmtQNFFieE9SVmdZVW9WTHlzbnBPZlNIdGVHOE1LNVNESjN3cGtVSHVpT1NJWHE1ZzNmUTVTOC0xX3NGSmJqU19IbjZfQWtMRG1YNUQtRy13TUJIZFlyOXJkQzFQbkdZVXVzM2czbS1HWHFBT1pXdVd3N09tcG82SVhnY1ZtUWxqTEg2UzJCUmllb2pweVN2aGwwS1FVRUhjNEN2amRMc3MwVU4zN3dVMWM5Slg4SERtenFaQk1yMWx0LWtxVWtLZVVtbU4yejVEM2h6TEt0RGdfWE09", + "provider": "1b59e032-3eb6-4694-93a5-df84cd9b3ce2" + } } ] diff --git a/src/backend/api/fixtures/3_dev_scans.json b/src/backend/api/fixtures/3_dev_scans.json index f37e2bb540..f9f3b04735 100644 --- a/src/backend/api/fixtures/3_dev_scans.json +++ b/src/backend/api/fixtures/3_dev_scans.json @@ -9,16 +9,16 @@ "name": "test scan 1", "state": "completed", "unique_resource_count": 1, - "duration": 10, + "duration": 5, "scanner_args": { - "key1": "value1", - "key2": { - "key21": "value21" - } + "checks_to_execute": [ + "accessanalyzer_enabled" + ] }, - "scheduled_at": "2024-09-01T17:20:27.050Z", "inserted_at": "2024-09-01T17:24:27.050Z", - "updated_at": "2024-09-01T17:24:27.050Z" + "started_at": "2024-09-01T17:24:27.050Z", + "updated_at": "2024-09-01T17:24:27.050Z", + "completed_at": "2024-09-01T17:24:37.050Z" } }, { @@ -31,16 +31,38 @@ "name": "test aws scan 2", "state": "completed", "unique_resource_count": 1, + "duration": 20, + "scanner_args": { + "checks_to_execute": [ + "accessanalyzer_enabled" + ] + }, + "inserted_at": "2024-09-02T17:24:27.050Z", + "started_at": "2024-09-02T17:24:27.050Z", + "updated_at": "2024-09-02T17:24:27.050Z", + "completed_at": "2024-09-01T17:24:37.050Z" + } + }, + { + "model": "api.scan", + "pk": "01920573-ea5b-77fd-a93f-1ed2ae12f728", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "provider": "baa7b895-8bac-4f47-b010-4226d132856e", + "trigger": "manual", + "name": "test gcp scan", + "state": "completed", + "unique_resource_count": 10, "duration": 10, "scanner_args": { - "key1": "value1", - "key2": { - "key21": "value21" - } + "checks_to_execute": [ + "cloudsql_instance_automated_backups" + ] }, - "scheduled_at": "2024-09-02T17:20:27.050Z", - "inserted_at": "2024-09-02T17:24:27.050Z", - "updated_at": "2024-09-02T17:24:27.050Z" + "inserted_at": "2024-09-02T19:24:27.050Z", + "started_at": "2024-09-02T19:24:27.050Z", + "updated_at": "2024-09-02T19:24:27.050Z", + "completed_at": "2024-09-01T17:24:37.050Z" } }, { @@ -53,14 +75,93 @@ "name": "test aws scan", "state": "completed", "unique_resource_count": 1, - "duration": 10, + "duration": 35, "scanner_args": { - "key1": "value1", - "key2": { - "key21": "value21" - } + "checks_to_execute": [ + "accessanalyzer_enabled" + ] }, - "scheduled_at": "2024-09-02T19:20:27.050Z", + "inserted_at": "2024-09-02T19:24:27.050Z", + "started_at": "2024-09-02T19:24:27.050Z", + "updated_at": "2024-09-02T19:24:27.050Z", + "completed_at": "2024-09-01T17:24:37.050Z" + } + }, + { + "model": "api.scan", + "pk": "c281c924-23f3-4fcc-ac63-73a22154b7de", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "provider": "b85601a8-4b45-4194-8135-03fb980ef428", + "trigger": "scheduled", + "name": "test scheduled aws scan", + "state": "available", + "scanner_args": { + "checks_to_execute": [ + "cloudformation_stack_outputs_find_secrets" + ] + }, + "scheduled_at": "2030-09-02T19:20:27.050Z", + "inserted_at": "2024-09-02T19:24:27.050Z", + "updated_at": "2024-09-02T19:24:27.050Z" + } + }, + { + "model": "api.scan", + "pk": "25c8907c-b26e-4ec0-966b-a1f53a39d8e6", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "provider": "b85601a8-4b45-4194-8135-03fb980ef428", + "trigger": "scheduled", + "name": "test scheduled aws scan 2", + "state": "available", + "scanner_args": { + "checks_to_execute": [ + "accessanalyzer_enabled", + "cloudformation_stack_outputs_find_secrets" + ] + }, + "scheduled_at": "2030-08-02T19:20:27.050Z", + "inserted_at": "2024-09-02T19:24:27.050Z", + "updated_at": "2024-09-02T19:24:27.050Z" + } + }, + { + "model": "api.scan", + "pk": "25c8907c-b26e-4ec0-966b-a1f53a39d8e6", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "provider": "baa7b895-8bac-4f47-b010-4226d132856e", + "trigger": "scheduled", + "name": "test scheduled gcp scan", + "state": "available", + "scanner_args": { + "checks_to_execute": [ + "cloudsql_instance_automated_backups", + "iam_audit_logs_enabled" + ] + }, + "scheduled_at": "2030-07-02T19:20:27.050Z", + "inserted_at": "2024-09-02T19:24:27.050Z", + "updated_at": "2024-09-02T19:24:27.050Z" + } + }, + { + "model": "api.scan", + "pk": "25c8907c-b26e-4ec0-966b-a1f53a39d8e6", + "fields": { + "tenant": "12646005-9067-4d2a-a098-8bb378604362", + "provider": "1b59e032-3eb6-4694-93a5-df84cd9b3ce2", + "trigger": "scheduled", + "name": "test scheduled azure scan", + "state": "available", + "scanner_args": { + "checks_to_execute": [ + "aks_cluster_rbac_enabled", + "defender_additional_email_configured_with_a_security_contact" + ] + }, + "scheduled_at": "2030-08-05T19:20:27.050Z", "inserted_at": "2024-09-02T19:24:27.050Z", "updated_at": "2024-09-02T19:24:27.050Z" } diff --git a/src/backend/api/migrations/0001_initial.py b/src/backend/api/migrations/0001_initial.py index aba4eae2a1..ca0caaf830 100644 --- a/src/backend/api/migrations/0001_initial.py +++ b/src/backend/api/migrations/0001_initial.py @@ -27,6 +27,8 @@ from api.db_utils import ( MemberRoleEnum, ProviderEnum, ProviderEnumField, + ProviderSecretTypeEnum, + ProviderSecretTypeEnumField, ScanTriggerEnum, StateEnumField, StateEnum, @@ -46,6 +48,7 @@ from api.models import ( StatusChoices, SeverityChoices, Membership, + ProviderSecret, ) DB_NAME = settings.DATABASES["default"]["NAME"] @@ -87,6 +90,13 @@ SeverityEnumMigration = PostgresEnumMigration( enum_values=tuple(severity[0] for severity in SeverityChoices), ) +ProviderSecretTypeEnumMigration = PostgresEnumMigration( + enum_name="provider_secret_type", + enum_values=tuple( + secret_type[0] for secret_type in ProviderSecret.TypeChoices.choices + ), +) + class Migration(migrations.Migration): initial = True @@ -1123,4 +1133,68 @@ class Migration(migrations.Migration): name="finding", options={}, ), + migrations.RunPython( + ProviderSecretTypeEnumMigration.create_enum_type, + reverse_code=ProviderSecretTypeEnumMigration.drop_enum_type, + ), + migrations.RunPython(partial(register_enum, enum_class=ProviderSecretTypeEnum)), + migrations.CreateModel( + name="ProviderSecret", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "name", + models.CharField( + blank=True, + max_length=100, + null=True, + validators=[django.core.validators.MinLengthValidator(3)], + ), + ), + ( + "secret_type", + ProviderSecretTypeEnumField( + choices=ProviderSecret.TypeChoices.choices + ), + ), + ("_secret", models.BinaryField(db_column="secret")), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ( + "provider", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="secret", + related_query_name="secret", + to="api.provider", + ), + ), + ], + options={ + "db_table": "provider_secrets", + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="providersecret", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_providersecret", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), ] diff --git a/src/backend/api/models.py b/src/backend/api/models.py index b0446684b9..303cc96aa8 100644 --- a/src/backend/api/models.py +++ b/src/backend/api/models.py @@ -1,6 +1,9 @@ +import json import re from uuid import uuid4, UUID +from cryptography.fernet import Fernet +from django.conf import settings from django.contrib.auth.models import AbstractBaseUser from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.search import SearchVector, SearchVectorField @@ -23,6 +26,7 @@ from api.db_utils import ( SeverityEnumField, StatusEnumField, CustomUserManager, + ProviderSecretTypeEnumField, ) from api.exceptions import ModelValidationError from api.rls import ( @@ -34,6 +38,8 @@ from api.rls import ( BaseSecurityConstraint, ) +fernet = Fernet(settings.SECRETS_ENCRYPTION_KEY.encode()) + # Convert Prowler Severity enum to Django TextChoices SeverityChoices = enum_to_choices(Severity) @@ -605,3 +611,50 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] + + +class ProviderSecret(RowLevelSecurityProtectedModel): + class TypeChoices(models.TextChoices): + STATIC = "static", _("Key-value pairs") + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + name = models.CharField( + blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)] + ) + secret_type = ProviderSecretTypeEnumField(choices=TypeChoices.choices) + _secret = models.BinaryField(db_column="secret") + provider = models.OneToOneField( + Provider, + on_delete=models.CASCADE, + related_name="secret", + related_query_name="secret", + ) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "provider_secrets" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + @property + def secret(self): + if isinstance(self._secret, memoryview): + encrypted_bytes = self._secret.tobytes() + elif isinstance(self._secret, str): + encrypted_bytes = self._secret.encode() + else: + encrypted_bytes = self._secret + decrypted_data = fernet.decrypt(encrypted_bytes) + return json.loads(decrypted_data.decode()) + + @secret.setter + def secret(self, value): + encrypted_data = fernet.encrypt(json.dumps(value).encode()) + self._secret = encrypted_data diff --git a/src/backend/api/specs/v1.yaml b/src/backend/api/specs/v1.yaml index 22dd9adaaf..64e2eee3db 100644 --- a/src/backend/api/specs/v1.yaml +++ b/src/backend/api/specs/v1.yaml @@ -525,6 +525,7 @@ paths: - alias - connection - scanner_args + - secret - url description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. @@ -734,6 +735,7 @@ paths: - alias - connection - scanner_args + - secret - url description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. @@ -840,6 +842,215 @@ paths: schema: $ref: '#/components/schemas/OpenApiResponseResponse' description: '' + /api/v1/providers/secrets: + get: + operationId: providers_secrets_list + description: Retrieve a list of all secrets with options for filtering by various + criteria. + summary: List all secrets + parameters: + - in: query + name: fields[ProviderSecret] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - name + - provider + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[name] + schema: + type: string + - in: query + name: filter[name__icontains] + schema: + type: string + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - name + - -name + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Provider + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedProviderSecretList' + description: '' + post: + operationId: providers_secrets_create + description: Add a new secret to the system by providing the required secret + details. + summary: Create a new secret + tags: + - Provider + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ProviderSecretCreateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/ProviderSecretCreateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/ProviderSecretCreateRequest' + required: true + security: + - jwtAuth: [] + responses: + '201': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ProviderSecretCreateResponse' + description: '' + /api/v1/providers/secrets/{id}: + get: + operationId: providers_secrets_retrieve + description: Fetch detailed information about a specific secret by their ID. + summary: Retrieve data from a secret + parameters: + - in: query + name: fields[ProviderSecret] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - name + - provider + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + required: true + tags: + - Provider + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ProviderSecretResponse' + description: '' + patch: + operationId: providers_secrets_partial_update + description: Update certain fields of an existing secret's information without + affecting other fields. + summary: Partially update a secret + parameters: + - in: path + name: id + schema: + type: string + format: uuid + required: true + tags: + - Provider + requestBody: + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PatchedProviderSecretUpdateRequest' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PatchedProviderSecretUpdateRequest' + multipart/form-data: + schema: + $ref: '#/components/schemas/PatchedProviderSecretUpdateRequest' + required: true + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ProviderSecretUpdateResponse' + description: '' + delete: + operationId: providers_secrets_destroy + description: Remove a secret from the system by their ID. + summary: Delete a secret + parameters: + - in: path + name: id + schema: + type: string + format: uuid + required: true + tags: + - Provider + security: + - jwtAuth: [] + responses: + '204': + description: No response body /api/v1/resources: get: operationId: resources_list @@ -2651,6 +2862,15 @@ components: $ref: '#/components/schemas/Provider' required: - data + PaginatedProviderSecretList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/ProviderSecret' + required: + - data PaginatedResourceList: type: object properties: @@ -2687,6 +2907,74 @@ components: $ref: '#/components/schemas/Tenant' required: - data + PatchedProviderSecretUpdateRequest: + type: object + properties: + data: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - ProviderSecret + id: {} + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + nullable: true + maxLength: 100 + minLength: 3 + secret: + writeOnly: true + required: + - secret + relationships: + type: object + properties: + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - Provider + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share + common attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + required: + - data PatchedProviderUpdateRequest: type: object properties: @@ -2915,6 +3203,35 @@ components: required: - provider - uid + relationships: + type: object + properties: + secret: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - ProviderSecret + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + required: + - secret ProviderCreate: type: object required: @@ -3015,6 +3332,370 @@ components: $ref: '#/components/schemas/Provider' required: - data + ProviderSecret: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/Type374Enum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + nullable: true + maxLength: 100 + minLength: 3 + relationships: + type: object + properties: + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - Provider + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + required: + - provider + ProviderSecretCreate: + type: object + required: + - type + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/Type374Enum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + nullable: true + maxLength: 100 + minLength: 3 + secret_type: + enum: + - static + type: string + description: '* `static` - Key-value pairs' + secret: + oneOf: + - type: object + title: AWS static credentials + properties: + aws_access_key_id: + type: string + aws_secret_access_key: + type: string + aws_session_token: + type: string + required: + - aws_access_key_id + - aws_secret_access_key + - type: object + title: Azure static credentials + properties: + client_id: + type: string + client_secret: + type: string + tenant_id: + type: string + description: Azure tenant ID + required: + - client_id + - client_secret + - tenant_id + - type: object + title: GCP static credentials + properties: + client_id: + type: string + client_secret: + type: string + refresh_token: + type: string + required: + - client_id + - client_secret + - refresh_token + writeOnly: true + required: + - secret_type + - secret + relationships: + type: object + properties: + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - Provider + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + required: + - provider + ProviderSecretCreateRequest: + type: object + properties: + data: + type: object + required: + - type + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - ProviderSecret + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + nullable: true + maxLength: 100 + minLength: 3 + secret_type: + enum: + - static + type: string + description: '* `static` - Key-value pairs' + secret: + oneOf: + - type: object + title: AWS static credentials + properties: + aws_access_key_id: + type: string + aws_secret_access_key: + type: string + aws_session_token: + type: string + required: + - aws_access_key_id + - aws_secret_access_key + - type: object + title: Azure static credentials + properties: + client_id: + type: string + client_secret: + type: string + tenant_id: + type: string + description: Azure tenant ID + required: + - client_id + - client_secret + - tenant_id + - type: object + title: GCP static credentials + properties: + client_id: + type: string + client_secret: + type: string + refresh_token: + type: string + required: + - client_id + - client_secret + - refresh_token + writeOnly: true + required: + - secret_type + - secret + relationships: + type: object + properties: + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - Provider + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share + common attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + required: + - provider + required: + - data + ProviderSecretCreateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ProviderSecretCreate' + required: + - data + ProviderSecretResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ProviderSecret' + required: + - data + ProviderSecretUpdate: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/Type374Enum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + name: + type: string + nullable: true + maxLength: 100 + minLength: 3 + secret: + writeOnly: true + required: + - secret + relationships: + type: object + properties: + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - Provider + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + ProviderSecretUpdateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ProviderSecretUpdate' + required: + - data Resource: type: object required: @@ -3705,6 +4386,10 @@ components: type: string enum: - Scan + Type374Enum: + type: string + enum: + - ProviderSecret Type4e8Enum: type: string enum: diff --git a/src/backend/api/tests/test_utils.py b/src/backend/api/tests/test_utils.py index 4ff5225ac9..555841908e 100644 --- a/src/backend/api/tests/test_utils.py +++ b/src/backend/api/tests/test_utils.py @@ -1,4 +1,18 @@ -from api.utils import merge_dicts +from unittest.mock import MagicMock, patch + +import pytest +from prowler.providers.aws.aws_provider import AwsProvider +from prowler.providers.azure.azure_provider import AzureProvider +from prowler.providers.gcp.gcp_provider import GcpProvider +from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider + +from api.models import Provider +from api.utils import ( + merge_dicts, + return_prowler_provider, + initialize_prowler_provider, + prowler_provider_connection_test, +) class TestMergeDicts: @@ -74,3 +88,51 @@ class TestMergeDicts: result = merge_dicts(default_dict, replacement_dict) assert result == expected_result + + +class TestReturnProwlerProvider: + @pytest.mark.parametrize( + "provider_type, expected_provider", + [ + (Provider.ProviderChoices.AWS.value, AwsProvider), + (Provider.ProviderChoices.GCP.value, GcpProvider), + (Provider.ProviderChoices.AZURE.value, AzureProvider), + (Provider.ProviderChoices.KUBERNETES.value, KubernetesProvider), + ], + ) + def test_return_prowler_provider(self, provider_type, expected_provider): + provider = MagicMock() + provider.provider = provider_type + prowler_provider = return_prowler_provider(provider) + assert prowler_provider == expected_provider + + def test_return_prowler_provider_unsupported_provider(self): + provider = MagicMock() + provider.provider = "UNSUPPORTED_PROVIDER" + with pytest.raises(ValueError): + return return_prowler_provider(provider) + + +class TestInitializeProwlerProvider: + @patch("api.utils.return_prowler_provider") + def test_initialize_prowler_provider(self, mock_return_prowler_provider): + provider = MagicMock() + provider.secret.secret = {"key": "value"} + mock_return_prowler_provider.return_value = MagicMock() + + initialize_prowler_provider(provider) + mock_return_prowler_provider.return_value.assert_called_once_with(key="value") + + +class TestProwlerProviderConnectionTest: + @patch("api.utils.return_prowler_provider") + def test_prowler_provider_connection_test(self, mock_return_prowler_provider): + provider = MagicMock() + provider.uid = "1234567890" + provider.secret.secret = {"key": "value"} + mock_return_prowler_provider.return_value = MagicMock() + + prowler_provider_connection_test(provider) + mock_return_prowler_provider.return_value.test_connection.assert_called_once_with( + key="value", provider_id="1234567890", raise_on_exception=False + ) diff --git a/src/backend/api/tests/test_views.py b/src/backend/api/tests/test_views.py index 639702d81c..13c9f970bc 100644 --- a/src/backend/api/tests/test_views.py +++ b/src/backend/api/tests/test_views.py @@ -1,3 +1,4 @@ +import json from datetime import datetime from unittest.mock import Mock, patch, ANY @@ -5,7 +6,7 @@ import pytest from django.urls import reverse from rest_framework import status -from api.models import User, Membership, Provider, Scan +from api.models import User, Membership, Provider, Scan, ProviderSecret from api.rls import Tenant from conftest import ( API_JSON_CONTENT_TYPE, @@ -1074,6 +1075,298 @@ class TestProviderViewSet: assert response.status_code == status.HTTP_400_BAD_REQUEST +@pytest.mark.django_db +class TestProviderSecretViewSet: + def test_provider_secrets_list(self, authenticated_client, provider_secret_fixture): + response = authenticated_client.get(reverse("providersecret-list")) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == len(provider_secret_fixture) + + def test_provider_secrets_retrieve( + self, authenticated_client, provider_secret_fixture + ): + provider_secret1, *_ = provider_secret_fixture + response = authenticated_client.get( + reverse("providersecret-detail", kwargs={"pk": provider_secret1.id}), + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["data"]["attributes"]["name"] == provider_secret1.name + assert ( + response.json()["data"]["attributes"]["secret_type"] + == provider_secret1.secret_type + ) + + def test_provider_secrets_invalid_retrieve(self, authenticated_client): + response = authenticated_client.get( + reverse( + "providersecret-detail", + kwargs={"pk": "f498b103-c760-4785-9a3e-e23fafbb7b02"}, + ) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_provider_secrets_create_valid( + self, authenticated_client, providers_fixture + ): + provider, *_ = providers_fixture + data = { + "data": { + "type": "ProviderSecret", + "attributes": { + "name": "My Secret", + "secret_type": "static", + "secret": { + "aws_access_key_id": "value", + "aws_secret_access_key": "value", + "aws_session_token": "value", + }, + }, + "relationships": { + "provider": {"data": {"type": "Provider", "id": str(provider.id)}} + }, + } + } + response = authenticated_client.post( + reverse("providersecret-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_201_CREATED + assert ProviderSecret.objects.count() == 1 + assert ProviderSecret.objects.get().name == data["data"]["attributes"]["name"] + assert ( + ProviderSecret.objects.get().secret_type + == data["data"]["attributes"]["secret_type"] + ) + assert ( + str(ProviderSecret.objects.get().provider.id) + == data["data"]["relationships"]["provider"]["data"]["id"] + ) + + @pytest.mark.parametrize( + "attributes, error_code, error_pointer", + ( + [ + ( + { + "name": "testing", + "secret_type": "static", + "secret": {"invalid": "test"}, + }, + "required", + "secret/aws_access_key_id", + ), + ( + { + "name": "testing", + "secret_type": "invalid", + "secret": {"invalid": "test"}, + }, + "invalid_choice", + "secret_type", + ), + ( + { + "name": "a" * 151, + "secret_type": "static", + "secret": { + "aws_access_key_id": "value", + "aws_secret_access_key": "value", + "aws_session_token": "value", + }, + }, + "max_length", + "name", + ), + ] + ), + ) + def test_provider_secrets_invalid_create( + self, + providers_fixture, + authenticated_client, + attributes, + error_code, + error_pointer, + ): + provider, *_ = providers_fixture + data = { + "data": { + "type": "ProviderSecret", + "attributes": attributes, + "relationships": { + "provider": {"data": {"type": "Provider", "id": str(provider.id)}} + }, + } + } + response = authenticated_client.post( + reverse("providersecret-list"), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json()["errors"][0]["code"] == error_code + assert ( + response.json()["errors"][0]["source"]["pointer"] + == f"/data/attributes/{error_pointer}" + ) + + def test_provider_secrets_partial_update( + self, authenticated_client, provider_secret_fixture + ): + provider_secret, *_ = provider_secret_fixture + data = { + "data": { + "type": "ProviderSecret", + "id": str(provider_secret.id), + "attributes": { + "name": "new_name", + "secret": { + "aws_access_key_id": "new_value", + "aws_secret_access_key": "new_value", + "aws_session_token": "new_value", + }, + }, + "relationships": { + "provider": { + "data": { + "type": "Provider", + "id": str(provider_secret.provider.id), + } + } + }, + } + } + response = authenticated_client.patch( + reverse("providersecret-detail", kwargs={"pk": provider_secret.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_200_OK + provider_secret.refresh_from_db() + assert provider_secret.name == "new_name" + for value in provider_secret.secret.values(): + assert value == "new_value" + + def test_provider_secrets_partial_update_invalid_content_type( + self, authenticated_client, provider_secret_fixture + ): + provider_secret, *_ = provider_secret_fixture + response = authenticated_client.patch( + reverse("providersecret-detail", kwargs={"pk": provider_secret.id}), + data={}, + ) + assert response.status_code == status.HTTP_415_UNSUPPORTED_MEDIA_TYPE + + def test_provider_secrets_partial_update_invalid_content( + self, authenticated_client, provider_secret_fixture + ): + provider_secret, *_ = provider_secret_fixture + data = { + "data": { + "type": "ProviderSecret", + "id": str(provider_secret.id), + "attributes": {"invalid_secret": "value"}, + "relationships": { + "provider": { + "data": { + "type": "Provider", + "id": str(provider_secret.provider.id), + } + } + }, + } + } + response = authenticated_client.patch( + reverse("providersecret-detail", kwargs={"pk": provider_secret.id}), + data=json.dumps(data), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_provider_secrets_delete( + self, + authenticated_client, + provider_secret_fixture, + ): + provider_secret, *_ = provider_secret_fixture + response = authenticated_client.delete( + reverse("providersecret-detail", kwargs={"pk": provider_secret.id}) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + + def test_provider_secrets_delete_invalid(self, authenticated_client): + response = authenticated_client.delete( + reverse( + "providersecret-detail", + kwargs={"pk": "e67d0283-440f-48d1-b5f8-38d0763474f4"}, + ) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + @pytest.mark.parametrize( + "filter_name, filter_value, expected_count", + ( + [ + ("name", "aws_testing_1", 1), + ("name.icontains", "aws", 2), + ] + ), + ) + def test_provider_secrets_filters( + self, + authenticated_client, + provider_secret_fixture, + filter_name, + filter_value, + expected_count, + ): + response = authenticated_client.get( + reverse("providersecret-list"), + {f"filter[{filter_name}]": filter_value}, + ) + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == expected_count + + @pytest.mark.parametrize( + "filter_name", + ( + [ + "invalid", + ] + ), + ) + def test_provider_secrets_filters_invalid(self, authenticated_client, filter_name): + response = authenticated_client.get( + reverse("providersecret-list"), + {f"filter[{filter_name}]": "whatever"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @pytest.mark.parametrize( + "sort_field", + ( + [ + "name", + "inserted_at", + "updated_at", + ] + ), + ) + def test_provider_secrets_sort(self, authenticated_client, sort_field): + response = authenticated_client.get( + reverse("providersecret-list"), {"sort": sort_field} + ) + assert response.status_code == status.HTTP_200_OK + + def test_provider_secrets_sort_invalid(self, authenticated_client): + response = authenticated_client.get( + reverse("providersecret-list"), {"sort": "invalid"} + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @pytest.mark.django_db class TestScanViewSet: def test_scans_list(self, authenticated_client, scans_fixture): diff --git a/src/backend/api/utils.py b/src/backend/api/utils.py index c678ae9683..86c6c529d8 100644 --- a/src/backend/api/utils.py +++ b/src/backend/api/utils.py @@ -1,3 +1,12 @@ +from prowler.providers.aws.aws_provider import AwsProvider +from prowler.providers.azure.azure_provider import AzureProvider +from prowler.providers.common.models import Connection +from prowler.providers.gcp.gcp_provider import GcpProvider +from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider + +from api.models import Provider + + def merge_dicts(default_dict: dict, replacement_dict: dict) -> dict: """ Recursively merge two dictionaries, using `default_dict` as the base and `replacement_dict` for overriding values. @@ -23,3 +32,65 @@ def merge_dicts(default_dict: dict, replacement_dict: dict) -> dict: result[key] = value return result + + +def return_prowler_provider( + provider: Provider, +) -> [AwsProvider | AzureProvider | GcpProvider | KubernetesProvider]: + """Return the Prowler provider class based on the given provider type. + + Args: + provider (Provider): The provider object containing the provider type and associated secrets. + + Returns: + AwsProvider | AzureProvider | GcpProvider | KubernetesProvider: The corresponding provider class. + + Raises: + ValueError: If the provider type specified in `provider.provider` is not supported. + """ + match provider.provider: + case Provider.ProviderChoices.AWS.value: + prowler_provider = AwsProvider + case Provider.ProviderChoices.GCP.value: + prowler_provider = GcpProvider + case Provider.ProviderChoices.AZURE.value: + prowler_provider = AzureProvider + case Provider.ProviderChoices.KUBERNETES.value: + prowler_provider = KubernetesProvider + case _: + raise ValueError(f"Provider type {provider.provider} not supported") + return prowler_provider + + +def initialize_prowler_provider( + provider: Provider, +) -> AwsProvider | AzureProvider | GcpProvider | KubernetesProvider: + """Initialize a Prowler provider instance based on the given provider type. + + Args: + provider (Provider): The provider object containing the provider type and associated secrets. + + Returns: + AwsProvider | AzureProvider | GcpProvider | KubernetesProvider: An instance of the corresponding provider class + (`AwsProvider`, `AzureProvider`, `GcpProvider`, or `KubernetesProvider`) initialized with the + provider's secrets. + """ + prowler_provider = return_prowler_provider(provider) + prowler_provider_kwargs = provider.secret.secret + return prowler_provider(**prowler_provider_kwargs) + + +def prowler_provider_connection_test(provider: Provider) -> Connection: + """Test the connection to a Prowler provider based on the given provider type. + + Args: + provider (Provider): The provider object containing the provider type and associated secrets. + + Returns: + Connection: A connection object representing the result of the connection test for the specified provider. + """ + prowler_provider = return_prowler_provider(provider) + prowler_provider_kwargs = provider.secret.secret + return prowler_provider.test_connection( + **prowler_provider_kwargs, provider_id=provider.uid, raise_on_exception=False + ) diff --git a/src/backend/api/v1/serializers.py b/src/backend/api/v1/serializers.py index c3aa272e72..66bf0784eb 100644 --- a/src/backend/api/v1/serializers.py +++ b/src/backend/api/v1/serializers.py @@ -24,6 +24,7 @@ from api.models import ( StatusChoices, Finding, ResourceFindingMapping, + ProviderSecret, ) from api.rls import Tenant from api.utils import merge_dicts @@ -57,17 +58,15 @@ class TokenSerializer(serializers.Serializer): # Authenticate user user = authenticate(username=email, password=password) if user is None: - raise serializers.ValidationError("Invalid credentials") + raise ValidationError("Invalid credentials") if tenant_id: if not user.is_member_of_tenant(tenant_id): - raise serializers.ValidationError( - "Tenant does not exist or user is not a member." - ) + raise ValidationError("Tenant does not exist or user is not a member.") else: first_membership = user.memberships.order_by("date_joined").first() if first_membership is None: - raise serializers.ValidationError("User has no memberships.") + raise ValidationError("User has no memberships.") tenant_id = str(first_membership.tenant_id) # Generate tokens @@ -77,14 +76,14 @@ class TokenSerializer(serializers.Serializer): access = refresh.access_token except InvalidKeyError: # Handle invalid key error - raise serializers.ValidationError( + raise ValidationError( { "detail": "Token generation failed due to invalid key configuration. Provide valid " "DJANGO_TOKEN_SIGNING_KEY and DJANGO_TOKEN_VERIFYING_KEY in the environment." } ) except Exception as e: - raise serializers.ValidationError({"detail": str(e)}) + raise ValidationError({"detail": str(e)}) return {"access": str(access), "refresh": str(refresh)} @@ -120,7 +119,7 @@ class TokenRefreshSerializer(serializers.Serializer): return {"access": str(access_token), "refresh": str(refresh)} except TokenError: - raise serializers.ValidationError({"refresh": "Invalid or expired token"}) + raise ValidationError({"refresh": "Invalid or expired token"}) # Base @@ -361,6 +360,7 @@ class ProviderSerializer(RLSSerializer): "alias", "connection", "scanner_args", + "secret", "url", ] @@ -628,3 +628,177 @@ class FindingSerializer(RLSSerializer): def get_resources(self, obj): mappings = ResourceFindingMapping.objects.filter(finding=obj) return Resource.objects.filter(id__in={m.resource_id for m in mappings}) + + +# Provider secrets +class BaseWriteProviderSecretSerializer(BaseWriteSerializer): + @staticmethod + def validate_secret_based_on_provider(provider_type: str, secret: dict): + if provider_type == Provider.ProviderChoices.AWS.value: + serializer = AwsProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.AZURE.value: + serializer = AzureProviderSecret(data=secret) + elif provider_type == Provider.ProviderChoices.GCP.value: + serializer = GCPProviderSecret(data=secret) + else: + raise serializers.ValidationError( + {"provider": f"Provider type not supported {provider_type}"} + ) + try: + serializer.is_valid(raise_exception=True) + except ValidationError as validation_error: + # Customize the error message + details = validation_error.detail.copy() + for key, value in details.items(): + validation_error.detail[f"secret/{key}"] = value + del validation_error.detail[key] + raise validation_error + + +class AwsProviderSecret(serializers.Serializer): + aws_access_key_id = serializers.CharField() + aws_secret_access_key = serializers.CharField() + aws_session_token = serializers.CharField(required=False) + + class Meta: + resource_name = "ProviderSecret" + + +class AzureProviderSecret(serializers.Serializer): + client_id = serializers.CharField() + client_secret = serializers.CharField() + tenant_id = serializers.CharField() + + class Meta: + resource_name = "ProviderSecret" + + +class GCPProviderSecret(serializers.Serializer): + client_id = serializers.CharField() + client_secret = serializers.CharField() + refresh_token = serializers.CharField() + + class Meta: + resource_name = "ProviderSecret" + + +@extend_schema_field( + { + "oneOf": [ + { + "type": "object", + "title": "AWS static credentials", + "properties": { + "aws_access_key_id": {"type": "string"}, + "aws_secret_access_key": {"type": "string"}, + "aws_session_token": {"type": "string"}, + }, + "required": ["aws_access_key_id", "aws_secret_access_key"], + }, + { + "type": "object", + "title": "Azure static credentials", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "tenant_id": {"type": "string", "description": "Azure tenant ID"}, + }, + "required": [ + "client_id", + "client_secret", + "tenant_id", + ], + }, + { + "type": "object", + "title": "GCP static credentials", + "properties": { + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "refresh_token": {"type": "string"}, + }, + "required": [ + "client_id", + "client_secret", + "refresh_token", + ], + }, + ] + } +) +class ProviderSecretField(serializers.JSONField): + pass + + +class ProviderSecretSerializer(RLSSerializer): + """ + Serializer for the ProviderSecret model. + """ + + class Meta: + model = ProviderSecret + fields = [ + "id", + "inserted_at", + "updated_at", + "name", + "secret_type", + "provider", + "url", + ] + + +class ProviderSecretCreateSerializer(RLSSerializer, BaseWriteProviderSecretSerializer): + secret = ProviderSecretField(write_only=True) + + class Meta: + model = ProviderSecret + fields = [ + "inserted_at", + "updated_at", + "name", + "secret_type", + "secret", + "provider", + ] + extra_kwargs = { + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + } + + def validate(self, attrs): + provider = attrs.get("provider") + secret = attrs.get("secret") + + validated_attrs = super().validate(attrs) + self.validate_secret_based_on_provider(provider.provider, secret) + return validated_attrs + + +class ProviderSecretUpdateSerializer(BaseWriteProviderSecretSerializer): + secret = serializers.JSONField(write_only=True) + + class Meta: + model = ProviderSecret + fields = [ + "inserted_at", + "updated_at", + "name", + "secret_type", + "secret", + "provider", + ] + extra_kwargs = { + "inserted_at": {"read_only": True}, + "updated_at": {"read_only": True}, + "provider": {"read_only": True}, + "secret_type": {"read_only": True}, + } + + def validate(self, attrs): + provider = self.instance.provider + secret = attrs.get("secret") + + validated_attrs = super().validate(attrs) + self.validate_secret_based_on_provider(provider.provider, secret) + return validated_attrs diff --git a/src/backend/api/v1/urls.py b/src/backend/api/v1/urls.py index 0e7658b584..448668149e 100644 --- a/src/backend/api/v1/urls.py +++ b/src/backend/api/v1/urls.py @@ -15,6 +15,7 @@ from api.v1.views import ( TaskViewSet, ResourceViewSet, FindingViewSet, + ProviderSecretViewSet, ) router = routers.DefaultRouter(trailing_slash=False) @@ -22,6 +23,7 @@ router = routers.DefaultRouter(trailing_slash=False) router.register(r"users", UserViewSet, basename="user") router.register(r"tenants", TenantViewSet, basename="tenant") router.register(r"providers", ProviderViewSet, basename="provider") +# router.register(r"providers/secrets", ProviderSecretViewSet, basename="provider-secret") router.register(r"scans", ScanViewSet, basename="scan") router.register(r"tasks", TaskViewSet, basename="task") router.register(r"resources", ResourceViewSet, basename="resource") @@ -38,6 +40,18 @@ users_router.register(r"memberships", MembershipViewSet, basename="user-membersh urlpatterns = [ path("tokens", CustomTokenObtainView.as_view(), name="token-obtain"), path("tokens/refresh", CustomTokenRefreshView.as_view(), name="token-refresh"), + path( + "providers/secrets", + ProviderSecretViewSet.as_view({"get": "list", "post": "create"}), + name="providersecret-list", + ), + path( + "providers/secrets/", + ProviderSecretViewSet.as_view( + {"get": "retrieve", "patch": "partial_update", "delete": "destroy"} + ), + name="providersecret-detail", + ), path("", include(router.urls)), path("", include(tenants_router.urls)), path("", include(users_router.urls)), diff --git a/src/backend/api/v1/views.py b/src/backend/api/v1/views.py index 6f9ed6bb43..4f692ea9a7 100644 --- a/src/backend/api/v1/views.py +++ b/src/backend/api/v1/views.py @@ -22,7 +22,6 @@ from rest_framework.generics import get_object_or_404, GenericAPIView from rest_framework_json_api.views import Response from rest_framework_simplejwt.exceptions import InvalidToken from rest_framework_simplejwt.exceptions import TokenError - from api.base_views import BaseTenantViewset, BaseRLSViewSet, BaseViewSet from api.filters import ( ProviderFilter, @@ -32,8 +31,18 @@ from api.filters import ( TaskFilter, ResourceFilter, FindingFilter, + ProviderSecretFilter, +) +from api.models import ( + User, + Membership, + Provider, + Scan, + Task, + Resource, + Finding, + ProviderSecret, ) -from api.models import User, Membership, Provider, Scan, Task, Resource, Finding from api.rls import Tenant from api.uuid_utils import datetime_to_uuid7 from api.v1.serializers import ( @@ -53,6 +62,9 @@ from api.v1.serializers import ( ScanUpdateSerializer, ResourceSerializer, FindingSerializer, + ProviderSecretSerializer, + ProviderSecretUpdateSerializer, + ProviderSecretCreateSerializer, ) from tasks.tasks import ( check_provider_connection_task, @@ -784,3 +796,56 @@ class FindingViewSet(BaseRLSViewSet): ).distinct() return queryset + + +@extend_schema_view( + list=extend_schema( + tags=["Provider"], + summary="List all secrets", + description="Retrieve a list of all secrets with options for filtering by various criteria.", + ), + retrieve=extend_schema( + tags=["Provider"], + summary="Retrieve data from a secret", + description="Fetch detailed information about a specific secret by their ID.", + ), + create=extend_schema( + tags=["Provider"], + summary="Create a new secret", + description="Add a new secret to the system by providing the required secret details.", + ), + partial_update=extend_schema( + tags=["Provider"], + summary="Partially update a secret", + description="Update certain fields of an existing secret's information without affecting other fields.", + ), + destroy=extend_schema( + tags=["Provider"], + summary="Delete a secret", + description="Remove a secret from the system by their ID.", + ), +) +@method_decorator(CACHE_DECORATOR, name="list") +@method_decorator(CACHE_DECORATOR, name="retrieve") +class ProviderSecretViewSet(BaseRLSViewSet): + queryset = ProviderSecret.objects.all() + serializer_class = ProviderSecretSerializer + filterset_class = ProviderSecretFilter + http_method_names = ["get", "post", "patch", "delete"] + search_fields = ["name"] + ordering = ["-inserted_at"] + ordering_fields = [ + "name", + "inserted_at", + "updated_at", + ] + + def get_queryset(self): + return ProviderSecret.objects.all() + + def get_serializer_class(self): + if self.action == "create": + return ProviderSecretCreateSerializer + elif self.action == "partial_update": + return ProviderSecretUpdateSerializer + return super().get_serializer_class() diff --git a/src/backend/config/django/base.py b/src/backend/config/django/base.py index 78abc6df49..8866f93c49 100644 --- a/src/backend/config/django/base.py +++ b/src/backend/config/django/base.py @@ -160,6 +160,8 @@ SIMPLE_JWT = { "TOKEN_REFRESH_SERIALIZER": "api.serializers.TokenRefreshSerializer", } +SECRETS_ENCRYPTION_KEY = env.str("DJANGO_SECRETS_ENCRYPTION_KEY", "") + # Internationalization # https://docs.djangoproject.com/en/5.0/topics/i18n/ diff --git a/src/backend/config/django/devel.py b/src/backend/config/django/devel.py index 9d135d25b5..573d913478 100644 --- a/src/backend/config/django/devel.py +++ b/src/backend/config/django/devel.py @@ -36,3 +36,5 @@ REST_FRAMEWORK["DEFAULT_FILTER_BACKENDS"] = tuple( # noqa: F405 for filter_backend in REST_FRAMEWORK["DEFAULT_FILTER_BACKENDS"] # noqa: F405 if "DjangoFilterBackend" not in filter_backend ) + ("api.filters.CustomDjangoFilterBackend",) + +SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0=" diff --git a/src/backend/config/django/testing.py b/src/backend/config/django/testing.py index ae374b6ef9..c62f0fc3a9 100644 --- a/src/backend/config/django/testing.py +++ b/src/backend/config/django/testing.py @@ -19,7 +19,7 @@ DATABASES = { DATABASE_ROUTERS = [] TESTING = True - +SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0=" # JWT diff --git a/src/backend/conftest.py b/src/backend/conftest.py index 3341b1fd45..eb9d62901e 100644 --- a/src/backend/conftest.py +++ b/src/backend/conftest.py @@ -21,6 +21,7 @@ from api.models import ( StateChoices, Task, Membership, + ProviderSecret, ) from api.rls import Tenant from api.v1.serializers import TokenSerializer @@ -160,6 +161,20 @@ def providers_fixture(tenants_fixture): return provider1, provider2, provider3, provider4, provider5 +@pytest.fixture +def provider_secret_fixture(providers_fixture): + return tuple( + ProviderSecret.objects.create( + tenant_id=provider.tenant_id, + provider=provider, + secret_type=ProviderSecret.TypeChoices.STATIC, + secret={"key": "value"}, + name=provider.alias, + ) + for provider in providers_fixture + ) + + @pytest.fixture def scans_fixture(tenants_fixture, providers_fixture): tenant, _ = tenants_fixture diff --git a/src/backend/tasks/jobs/connection.py b/src/backend/tasks/jobs/connection.py index feb189c61c..1583f8a75f 100644 --- a/src/backend/tasks/jobs/connection.py +++ b/src/backend/tasks/jobs/connection.py @@ -1,12 +1,9 @@ from datetime import datetime, timezone from celery.utils.log import get_task_logger -from prowler.providers.aws.aws_provider import AwsProvider -from prowler.providers.azure.azure_provider import AzureProvider -from prowler.providers.gcp.gcp_provider import GcpProvider -from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from api.models import Provider +from api.utils import prowler_provider_connection_test logger = get_task_logger(__name__) @@ -28,22 +25,8 @@ def check_provider_connection(provider_id: str): Model.DoesNotExist: If the provider does not exist. """ provider_instance = Provider.objects.get(pk=provider_id) - match provider_instance.provider: - # TODO Refactor when proper credentials are implemented - case Provider.ProviderChoices.AWS.value: - prowler_provider = AwsProvider - case Provider.ProviderChoices.GCP.value: - prowler_provider = GcpProvider - case Provider.ProviderChoices.AZURE.value: - prowler_provider = AzureProvider - case Provider.ProviderChoices.KUBERNETES.value: - prowler_provider = KubernetesProvider - case _: - raise ValueError( - f"Provider type {provider_instance.provider} not supported" - ) try: - connection_result = prowler_provider.test_connection(raise_on_exception=False) + connection_result = prowler_provider_connection_test(provider_instance) except Exception as e: logger.warning( f"Unexpected exception checking {provider_instance.provider} provider connection: {str(e)}" @@ -54,9 +37,5 @@ def check_provider_connection(provider_id: str): provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc) provider_instance.save() - connection_error = ( - f"{connection_result.error.__class__.__name__}: {connection_result.error}" - if connection_result.error - else None - ) + connection_error = f"{connection_result.error}" if connection_result.error else None return {"connected": connection_result.is_connected, "error": connection_error} diff --git a/src/backend/tasks/jobs/scan.py b/src/backend/tasks/jobs/scan.py index 2994122724..e583f45ac4 100644 --- a/src/backend/tasks/jobs/scan.py +++ b/src/backend/tasks/jobs/scan.py @@ -5,10 +5,6 @@ from datetime import datetime, timezone from celery.utils.log import get_task_logger from prowler.lib.outputs.finding import Finding as ProwlerFinding from prowler.lib.scan.scan import Scan as ProwlerScan -from prowler.providers.aws.aws_provider import AwsProvider -from prowler.providers.azure.azure_provider import AzureProvider -from prowler.providers.gcp.gcp_provider import GcpProvider -from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from api.db_utils import tenant_transaction from api.models import ( @@ -20,6 +16,7 @@ from api.models import ( StatusChoices as FindingStatus, StateChoices, ) +from api.utils import initialize_prowler_provider from api.v1.serializers import ScanTaskSerializer logger = get_task_logger(__name__) @@ -28,6 +25,19 @@ logger = get_task_logger(__name__) def _create_finding_delta( last_status: FindingStatus | None | str, new_status: FindingStatus | None ) -> Finding.DeltaChoices: + """ + Determine the delta status of a finding based on its previous and current status. + + Args: + last_status (FindingStatus | None | str): The previous status of the finding. Can be None or a string representation. + new_status (FindingStatus | None): The current status of the finding. + + Returns: + Finding.DeltaChoices: The delta status indicating if the finding is new, changed, or unchanged. + - Returns `Finding.DeltaChoices.NEW` if `last_status` is None. + - Returns `Finding.DeltaChoices.CHANGED` if `last_status` and `new_status` are different. + - Returns `None` if the status hasn't changed. + """ if last_status is None: return Finding.DeltaChoices.NEW return Finding.DeltaChoices.CHANGED if last_status != new_status else None @@ -39,6 +49,19 @@ def _store_finding( scan_instance: Scan, resource_instance: Resource, ) -> Finding: + """ + Store a finding in the database, calculate its delta status, and associate it with a resource and scan. + + Args: + finding (ProwlerFinding): The finding object obtained from the Prowler scan. + tenant_id (str): The ID of the tenant owning the finding. + scan_instance (Scan): The scan instance associated with the finding. + resource_instance (Resource): The resource instance associated with the finding. + + Returns: + Finding: The newly created or updated Finding instance. + + """ finding_uid = finding.finding_uid status = FindingStatus[finding.status.value] if finding.status is not None else None with tenant_transaction(tenant_id): @@ -70,6 +93,20 @@ def _store_finding( def _store_resources( finding: ProwlerFinding, tenant_id: str, provider_instance: Provider ) -> tuple[Resource, tuple[str, str]]: + """ + Store resource information from a finding, including tags, in the database. + + Args: + finding (ProwlerFinding): The finding object containing resource information. + tenant_id (str): The ID of the tenant owning the resource. + provider_instance (Provider): The provider instance associated with the resource. + + Returns: + tuple: + - Resource: The resource instance created or retrieved from the database. + - tuple[str, str]: A tuple containing the resource UID and region. + + """ with tenant_transaction(tenant_id): resource_instance, _ = Resource.objects.get_or_create( tenant_id=tenant_id, @@ -93,6 +130,22 @@ def _store_resources( def perform_prowler_scan( tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None ): + """ + Perform a scan using Prowler and store the findings and resources in the database. + + Args: + tenant_id (str): The ID of the tenant for which the scan is performed. + scan_id (str): The ID of the scan instance. + provider_id (str): The ID of the provider to scan. + checks_to_execute (list[str], optional): A list of specific checks to execute. Defaults to None. + + Returns: + dict: Serialized data of the completed scan instance. + + Raises: + ValueError: If the provider cannot be connected. + + """ with tenant_transaction(tenant_id): exception = None provider_instance = Provider.objects.get(pk=provider_id) @@ -104,32 +157,23 @@ def perform_prowler_scan( scan_instance.started_at = datetime.now(tz=timezone.utc) scan_instance.save() try: - match provider_instance.provider: - case Provider.ProviderChoices.AWS.value: - prowler_provider = AwsProvider - case Provider.ProviderChoices.GCP.value: - prowler_provider = GcpProvider - case Provider.ProviderChoices.AZURE.value: - prowler_provider = AzureProvider - case Provider.ProviderChoices.KUBERNETES.value: - prowler_provider = KubernetesProvider - case _: - raise ValueError( - f"Provider type {provider_instance.provider} not supported" - ) with tenant_transaction(tenant_id): - connection_status = prowler_provider.test_connection( - raise_on_exception=False - ) - provider_instance.connected = connection_status.is_connected - provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc) - provider_instance.save() - - if connection_status.is_connected is False: - raise ValueError(f"Provider {provider_instance.provider} is not connected") + try: + prowler_provider = initialize_prowler_provider(provider_instance) + provider_instance.connected = True + except Exception as e: + provider_instance.connected = False + raise ValueError( + f"Provider {provider_instance.provider} is not connected: {e}" + ) + finally: + provider_instance.connection_last_checked_at = datetime.now( + tz=timezone.utc + ) + provider_instance.save() prowler_scan = ProwlerScan( - provider=prowler_provider(), checks_to_execute=checks_to_execute or [] + provider=prowler_provider, checks_to_execute=checks_to_execute ) for progress, findings in prowler_scan.scan(): for finding in findings: diff --git a/src/backend/tasks/tasks.py b/src/backend/tasks/tasks.py index dbb6954383..e1db910576 100644 --- a/src/backend/tasks/tasks.py +++ b/src/backend/tasks/tasks.py @@ -22,7 +22,7 @@ def check_provider_connection_task(provider_id: str): - 'connected' (bool): Indicates whether the provider is successfully connected. - 'error' (str or None): The error message if the connection failed, otherwise `None`. """ - return check_provider_connection(provider_id) + return check_provider_connection(provider_id=provider_id) @shared_task(base=RLSTask, name="provider-deletion") diff --git a/src/backend/tasks/tests/test_connection.py b/src/backend/tasks/tests/test_connection.py index dad47eff13..75ba6dc2eb 100644 --- a/src/backend/tasks/tests/test_connection.py +++ b/src/backend/tasks/tests/test_connection.py @@ -8,35 +8,32 @@ from tasks.jobs.connection import check_provider_connection @pytest.mark.parametrize( - "provider_data, provider_class", + "provider_data", [ - ( - {"provider": "aws", "uid": "123456789012", "alias": "aws"}, - "AwsProvider", - ), + {"provider": "aws", "uid": "123456789012", "alias": "aws"}, ], ) +@patch("tasks.jobs.connection.prowler_provider_connection_test") @pytest.mark.django_db -def test_check_provider_connection(tenants_fixture, provider_data, provider_class): +def test_check_provider_connection( + mock_provider_connection_test, tenants_fixture, provider_data +): provider = Provider.objects.create(**provider_data, tenant_id=tenants_fixture[0].id) mock_test_connection_result = MagicMock() mock_test_connection_result.is_connected = True - with patch( - f"tasks.jobs.connection.{provider_class}.test_connection" - ) as mock_test_connection: - mock_test_connection.return_value = mock_test_connection_result + mock_provider_connection_test.return_value = mock_test_connection_result - check_provider_connection( - provider_id=str(provider.id), - ) - provider.refresh_from_db() + check_provider_connection( + provider_id=str(provider.id), + ) + provider.refresh_from_db() - mock_test_connection.assert_called_once() - assert provider.connected is True - assert provider.connection_last_checked_at is not None - assert provider.connection_last_checked_at <= datetime.now(tz=timezone.utc) + mock_provider_connection_test.assert_called_once() + assert provider.connected is True + assert provider.connection_last_checked_at is not None + assert provider.connection_last_checked_at <= datetime.now(tz=timezone.utc) @patch("tasks.jobs.connection.Provider.objects.get") @@ -53,16 +50,18 @@ def test_check_provider_connection_unsupported_provider(mock_provider_get): @patch("tasks.jobs.connection.Provider.objects.get") -@patch("tasks.jobs.connection.AwsProvider.test_connection") +@patch("tasks.jobs.connection.prowler_provider_connection_test") @pytest.mark.django_db -def test_check_provider_connection_exception(mock_test_connection, mock_provider_get): +def test_check_provider_connection_exception( + mock_provider_connection_test, mock_provider_get +): mock_provider_instance = MagicMock() mock_provider_instance.provider = Provider.ProviderChoices.AWS.value mock_provider_get.return_value = mock_provider_instance - mock_test_connection.return_value = MagicMock() - mock_test_connection.return_value.is_connected = False - mock_test_connection.return_value.error = Exception() + mock_provider_connection_test.return_value = MagicMock() + mock_provider_connection_test.return_value.is_connected = False + mock_provider_connection_test.return_value.error = Exception() result = check_provider_connection(provider_id="provider_id") diff --git a/src/backend/tasks/tests/test_scan.py b/src/backend/tasks/tests/test_scan.py index c389510066..225862c586 100644 --- a/src/backend/tasks/tests/test_scan.py +++ b/src/backend/tasks/tests/test_scan.py @@ -1,5 +1,5 @@ import json -from unittest.mock import patch, MagicMock, PropertyMock +from unittest.mock import patch, MagicMock import pytest @@ -10,12 +10,12 @@ from tasks.jobs.scan import perform_prowler_scan, _create_finding_delta @pytest.mark.django_db class TestPerformScan: @patch("tasks.jobs.scan.ProwlerScan") - @patch("tasks.jobs.scan.AwsProvider") + @patch("tasks.jobs.scan.initialize_prowler_provider") @patch("api.db_utils.tenant_transaction") def test_perform_prowler_scan_success( self, mock_tenant_transaction, - mock_aws_provider, + mock_prowler_provider, mock_prowler_scan, tenants_fixture, scans_fixture, @@ -33,12 +33,6 @@ class TestPerformScan: provider_id = str(provider.id) checks_to_execute = ["check1", "check2"] - connection_status_mock = MagicMock() - type(connection_status_mock).is_connected = PropertyMock(return_value=True) - mock_aws_provider.test_connection.return_value = connection_status_mock - mock_aws_provider_instance = mock_aws_provider.return_value - mock_aws_provider_instance.test_connection.return_value = connection_status_mock - finding = MagicMock() finding.finding_uid = "this_is_a_test_finding_id" finding.status = StatusChoices.PASS @@ -80,12 +74,12 @@ class TestPerformScan: assert scan_resource.type == finding.resource_type @patch("tasks.jobs.scan.ProwlerScan") - @patch("tasks.jobs.scan.AwsProvider") + @patch("tasks.jobs.scan.initialize_prowler_provider", side_effect=Exception) @patch("api.db_utils.tenant_transaction") def test_perform_prowler_scan_no_connection( self, mock_tenant_transaction, - mock_aws_provider, + mock_prowler_provider, mock_prowler_scan, tenants_fixture, scans_fixture, @@ -100,12 +94,6 @@ class TestPerformScan: provider_id = str(provider.id) checks_to_execute = ["check1", "check2"] - connection_status_mock = MagicMock() - type(connection_status_mock).is_connected = PropertyMock(return_value=False) - mock_aws_provider.test_connection.return_value = connection_status_mock - mock_aws_provider_instance = mock_aws_provider.return_value - mock_aws_provider_instance.test_connection.return_value = connection_status_mock - with pytest.raises(ValueError): perform_prowler_scan(tenant_id, scan_id, provider_id, checks_to_execute)