From ded28baa2f3cd6e2b6be6eb7731774dd014ca02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Fern=C3=A1ndez=20Poyatos?= Date: Thu, 3 Oct 2024 18:36:42 +0200 Subject: [PATCH] feat(Scan, Resource, Finding): PRWLR-4015 Implement on demand scans and findings/resources in database (#48) * feat(Users): PRWLR-4718 make user email the default login username * feat(Token): PRWLR-4718 add serializers, views and urls for access and refresh tokens * feat(Token): PRWLR-4718 add first membership tenant in token if not present in json body * feat(Users): PRWLR-4718 add company_name to model * feat(Users): PRWLR-4718 create tenant and membership when creating new user * fix(BaseView): PRWLR-4718 add tenant_id to serializer context * fix(Tests): PRWLR-4718 use authorization with unit tests * fix(Views): PRWLR-4718 fix tenant retrieval from request * fix(Tests): PRWLR-4718 fix tests * fix(Fixtures): PRWLR-4718 fix tenant memberships ordering * chore(Tokens): PRWLR-4718 update token url * chore(Spec): PRWLR-4718 update API spec * feat(Tokens): PRWLR-4718 enable token refresh blacklisting * feat(Tokens): PRWLR-4718 implement RS256 algorithm and dev valid keys * chore(env): PRWLR-4718 update .env.example * chore(Deps): PRWLR-4015 update prowler dep * fix(Resources, Findings): PRWLR-4015 fix permission issues on models and migrations * feat(Scans, Tasks, Resources, Findings): PRWLR-4015 perform whole flow when executing scans * fix(Settings): PRWLR-4015 fix devel JWT settings * chore(Scans, Tasks): PRWLR-4015 improve docs and responses format * test(Scan, Provider): PRWLR-4015 fix unit tests * chore(Environment): PRWLR-4015 fix .env.example values * test(Scan): PRWLR-4015 add unit tests for scan task * chore(Task): PRWLR-4015 give proper format to task result on scan perform * feat(Provider, Scan): PRWLR-4015 add all provider types to connection check and scan * fix(Logging): PRWLR-4015 fix API logger middleware to include tenant and user IDs --- .env.example | 5 +- poetry.lock | 231 ++++++++++++--------- src/backend/api/db_utils.py | 10 +- src/backend/api/middleware.py | 24 +-- src/backend/api/migrations/0001_initial.py | 151 +++++++------- src/backend/api/models.py | 119 ++++++----- src/backend/api/specs/v1.yaml | 136 +++++------- src/backend/api/tests/test_middleware.py | 69 +++--- src/backend/api/tests/test_views.py | 27 ++- src/backend/api/v1/serializers.py | 58 +++--- src/backend/api/v1/views.py | 96 ++++++--- src/backend/config/custom_logging.py | 12 +- src/backend/tasks/jobs/connection.py | 27 ++- src/backend/tasks/jobs/scan.py | 146 +++++++++++++ src/backend/tasks/tasks.py | 29 +++ src/backend/tasks/tests/test_scan.py | 111 ++++++++++ 16 files changed, 836 insertions(+), 415 deletions(-) create mode 100644 src/backend/tasks/jobs/scan.py create mode 100644 src/backend/tasks/tests/test_scan.py diff --git a/.env.example b/.env.example index 753ebedf0a..8698b5c225 100644 --- a/.env.example +++ b/.env.example @@ -11,10 +11,11 @@ DJANGO_LOGGING_FORMATTER=[ndjson|human_readable] # applies to both Django and Celery Workers DJANGO_LOGGING_LEVEL=INFO DJANGO_WORKERS=4 # Defaults to the maximum available based on CPU cores if not set. +DJANGO_TOKEN_SIGNING_KEY="" +DJANGO_TOKEN_VERIFYING_KEY="" # Token lifetime is in minutes DJANGO_ACCESS_TOKEN_LIFETIME=30 DJANGO_REFRESH_TOKEN_LIFETIME=1440 -DJANGO_TOKEN_SIGNING_KEY=S3cret DJANGO_CACHE_MAX_AGE=3600 DJANGO_STALE_WHILE_REVALIDATE=60 @@ -28,7 +29,7 @@ POSTGRES_USER=prowler_user POSTGRES_PASSWORD=S3cret POSTGRES_DB=prowler_db -# valkey settings. +# Valkey settings # If running django and celery on host, use localhost, else use 'valkey' VALKEY_HOST=[localhost|valkey] VALKEY_PORT=6379 diff --git a/poetry.lock b/poetry.lock index 61082a2705..26b79abe2a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -289,13 +289,13 @@ files = [ [[package]] name = "azure-core" -version = "1.30.2" +version = "1.31.0" description = "Microsoft Azure Core Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-core-1.30.2.tar.gz", hash = "sha256:a14dc210efcd608821aa472d9fb8e8d035d29b68993819147bc290a8ac224472"}, - {file = "azure_core-1.30.2-py3-none-any.whl", hash = "sha256:cf019c1ca832e96274ae85abd3d9f752397194d9fea3b41487290562ac8abe4a"}, + {file = "azure_core-1.31.0-py3-none-any.whl", hash = "sha256:22954de3777e0250029360ef31d80448ef1be13b80a459bff80ba7073379e2cd"}, + {file = "azure_core-1.31.0.tar.gz", hash = "sha256:656a0dd61e1869b1506b7c6a3b31d62f15984b1a573d6326f6aa2f3e4123284b"}, ] [package.dependencies] @@ -308,20 +308,20 @@ aio = ["aiohttp (>=3.0)"] [[package]] name = "azure-identity" -version = "1.17.1" +version = "1.18.0" description = "Microsoft Azure Identity Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-identity-1.17.1.tar.gz", hash = "sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea"}, - {file = "azure_identity-1.17.1-py3-none-any.whl", hash = "sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382"}, + {file = "azure_identity-1.18.0-py3-none-any.whl", hash = "sha256:bccf6106245b49ff41d0c4cd7b72851c5a2ba3a32cef7589da246f5727f26f02"}, + {file = "azure_identity-1.18.0.tar.gz", hash = "sha256:f567579a65d8932fa913c76eddf3305101a15e5727a5e4aa5df649a0f553d4c3"}, ] [package.dependencies] -azure-core = ">=1.23.0" +azure-core = ">=1.31.0" cryptography = ">=2.5" -msal = ">=1.24.0" -msal-extensions = ">=0.3.0" +msal = ">=1.30.0" +msal-extensions = ">=1.2.0" typing-extensions = ">=4.0.0" [[package]] @@ -390,15 +390,31 @@ azure-mgmt-core = ">=1.3.2" isodate = ">=0.6.1" typing-extensions = ">=4.6.0" +[[package]] +name = "azure-mgmt-containerregistry" +version = "10.3.0" +description = "Microsoft Azure Container Registry Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "azure-mgmt-containerregistry-10.3.0.tar.gz", hash = "sha256:ae21651855dfb19c42d91d6b3a965c6c611e23f8bc4bf7138835e652d2f918e3"}, + {file = "azure_mgmt_containerregistry-10.3.0-py3-none-any.whl", hash = "sha256:851e1c57f9bc4a3589c6b21fb627c11fd6cbb57a0388b7dfccd530ba3160805f"}, +] + +[package.dependencies] +azure-common = ">=1.1,<2.0" +azure-mgmt-core = ">=1.3.2,<2.0.0" +isodate = ">=0.6.1,<1.0.0" + [[package]] name = "azure-mgmt-containerservice" -version = "31.0.0" +version = "32.0.0" description = "Microsoft Azure Container Service Management Client Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-mgmt-containerservice-31.0.0.tar.gz", hash = "sha256:134358d7f88c4d29b4009f91d7619861e1fad5dbea5e147402dd61ad96b5624a"}, - {file = "azure_mgmt_containerservice-31.0.0-py3-none-any.whl", hash = "sha256:75c6dbbaf27ed6c8251e34b86fefcfb001fe096c69df9ce93c163de71a2b5275"}, + {file = "azure_mgmt_containerservice-32.0.0-py3-none-any.whl", hash = "sha256:66ff0a2651122207db0d5492e86550561f9d81509379abe7d401ed4805a1c66b"}, + {file = "azure_mgmt_containerservice-32.0.0.tar.gz", hash = "sha256:ccb587479d8a93ec78f7162590adc82e4fba4de76e4adc3419f33ad225efb292"}, ] [package.dependencies] @@ -423,19 +439,20 @@ azure-core = ">=1.26.2,<2.0.0" [[package]] name = "azure-mgmt-cosmosdb" -version = "9.5.1" +version = "9.6.0" description = "Microsoft Azure Cosmos DB Management Client Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-mgmt-cosmosdb-9.5.1.tar.gz", hash = "sha256:4e55d3973f11cf02cf7a8055aee86615233d7cbe3bb227a07c176bac8bc7ef02"}, - {file = "azure_mgmt_cosmosdb-9.5.1-py3-none-any.whl", hash = "sha256:860cad5583d63936e5f9477f7878a256c4dee039169cdea554da851f0762f0c7"}, + {file = "azure_mgmt_cosmosdb-9.6.0-py3-none-any.whl", hash = "sha256:02b4108867de58e0b89a206ee7b7588b439e1f6fef2377ce1979b803a0d02d5a"}, + {file = "azure_mgmt_cosmosdb-9.6.0.tar.gz", hash = "sha256:667c7d8a8f542b0e7972e63274af536ad985187e24a6cc2e3c8eef35560881fc"}, ] [package.dependencies] azure-common = ">=1.1" azure-mgmt-core = ">=1.3.2" isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" [[package]] name = "azure-mgmt-keyvault" @@ -472,19 +489,20 @@ isodate = ">=0.6.1,<1.0.0" [[package]] name = "azure-mgmt-network" -version = "26.0.0" +version = "27.0.0" description = "Microsoft Azure Network Management Client Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-mgmt-network-26.0.0.tar.gz", hash = "sha256:4de676184195053fdb106a6ea1042a894e70c731a6d3c8a633d52f5229f4ee1b"}, - {file = "azure_mgmt_network-26.0.0-py3-none-any.whl", hash = "sha256:15d6a77d7429bdcc76f8482158845d42f9d9434529e1ad2b6778269cc14627e9"}, + {file = "azure_mgmt_network-27.0.0-py3-none-any.whl", hash = "sha256:67295fda351fdae79d14591487009e911346adea9578bb1b04636a15e3472598"}, + {file = "azure_mgmt_network-27.0.0.tar.gz", hash = "sha256:5c1c61d8bb13ad40f788a26fd7569c1d9d60eb2e4cb19c2a1b5d9c02ae862316"}, ] [package.dependencies] azure-common = ">=1.1" azure-mgmt-core = ">=1.3.2" isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" [[package]] name = "azure-mgmt-rdbms" @@ -601,23 +619,23 @@ typing-extensions = ">=4.6.0" [[package]] name = "azure-storage-blob" -version = "12.22.0" +version = "12.23.1" description = "Microsoft Azure Blob Storage Client Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure-storage-blob-12.22.0.tar.gz", hash = "sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e"}, - {file = "azure_storage_blob-12.22.0-py3-none-any.whl", hash = "sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8"}, + {file = "azure_storage_blob-12.23.1-py3-none-any.whl", hash = "sha256:1c2238aa841d1545f42714a5017c010366137a44a0605da2d45f770174bfc6b4"}, + {file = "azure_storage_blob-12.23.1.tar.gz", hash = "sha256:a587e54d4e39d2a27bd75109db164ffa2058fe194061e5446c5a89bca918272f"}, ] [package.dependencies] -azure-core = ">=1.28.0" +azure-core = ">=1.30.0" cryptography = ">=2.1.4" isodate = ">=0.6.1" typing-extensions = ">=4.6.0" [package.extras] -aio = ["azure-core[aio] (>=1.28.0)"] +aio = ["azure-core[aio] (>=1.30.0)"] [[package]] name = "bandit" @@ -667,17 +685,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.162" +version = "1.35.28" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590"}, - {file = "boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f"}, + {file = "boto3-1.35.28-py3-none-any.whl", hash = "sha256:dc088b86a14f17d3cd2e96915c6ccfd31bce640dfe9180df579ed311bc6bf0fc"}, + {file = "boto3-1.35.28.tar.gz", hash = "sha256:8960fc458b9ba3c8a9890a607c31cee375db821f39aefaec9ff638248e81644a"}, ] [package.dependencies] -botocore = ">=1.34.162,<1.35.0" +botocore = ">=1.35.28,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -686,13 +704,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.162" +version = "1.35.29" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"}, - {file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"}, + {file = "botocore-1.35.29-py3-none-any.whl", hash = "sha256:f8e3ae0d84214eff3fb69cb4dc51cea6c43d3bde82027a94d00c52b941d6c3d5"}, + {file = "botocore-1.35.29.tar.gz", hash = "sha256:4ed28ab03675bb008a290c452c5ddd7aaa5d4e3fa1912aadbdf93057ee84362b"}, ] [package.dependencies] @@ -701,7 +719,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} [package.extras] -crt = ["awscrt (==0.21.2)"] +crt = ["awscrt (==0.21.5)"] [[package]] name = "cachetools" @@ -1149,13 +1167,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "dash" -version = "2.18.0" +version = "2.18.1" description = "A Python framework for building reactive web-apps. Developed by Plotly." optional = false python-versions = ">=3.8" files = [ - {file = "dash-2.18.0-py3-none-any.whl", hash = "sha256:2d37b479be20bef1a184d35be4a654e4da131da6538d563fc813be3e28b90cbc"}, - {file = "dash-2.18.0.tar.gz", hash = "sha256:9be6ea7562d40bb70211df8cbfb23611cc2ca9ae63cfca9af8f96679989adc40"}, + {file = "dash-2.18.1-py3-none-any.whl", hash = "sha256:07c4513bb5f79a4b936847a0b49afc21dbd4b001ff77ea78d4d836043e211a07"}, + {file = "dash-2.18.1.tar.gz", hash = "sha256:ffdf89690d734f6851ef1cb344222826ffb11ad2214ab9172668bf8aadd75d12"}, ] [package.dependencies] @@ -1618,6 +1636,16 @@ djangorestframework-jsonapi = ">=6.0.0" drf-extensions = ">=0.7.1" drf-spectacular = ">=0.25.0" +[[package]] +name = "durationpy" +version = "0.8" +description = "Module for converting between datetime.timedelta and Go's Duration strings." +optional = false +python-versions = "*" +files = [ + {file = "durationpy-0.8.tar.gz", hash = "sha256:e90941562e260bdb0e2244cd9eee9be5b3aa0efc0557fca754f36d18fe080f9d"}, +] + [[package]] name = "email-validator" version = "2.1.1" @@ -1810,13 +1838,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.144.0" +version = "2.147.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.144.0-py2.py3-none-any.whl", hash = "sha256:f9c333ac4454a012adca90c297f9a22611a8953f3aae5481f90b3a56b9bdd413"}, - {file = "google_api_python_client-2.144.0.tar.gz", hash = "sha256:fe00851b257157bca600e1692ed8a54762c4a5c7d9eb7f6d4822059424b0d0a9"}, + {file = "google_api_python_client-2.147.0-py2.py3-none-any.whl", hash = "sha256:c6ecfa193c695baa41e84562d8f8f244fcd164419eca3fc9fd7565646668f9b2"}, + {file = "google_api_python_client-2.147.0.tar.gz", hash = "sha256:e864c2cf61d34c00f05278b8bdb72b93b6fa34f0de9ead51d20435f3b65f91be"}, ] [package.dependencies] @@ -2212,17 +2240,18 @@ zookeeper = ["kazoo (>=2.8.0)"] [[package]] name = "kubernetes" -version = "30.1.0" +version = "31.0.0" description = "Kubernetes python client" optional = false python-versions = ">=3.6" files = [ - {file = "kubernetes-30.1.0-py2.py3-none-any.whl", hash = "sha256:e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d"}, - {file = "kubernetes-30.1.0.tar.gz", hash = "sha256:41e4c77af9f28e7a6c314e3bd06a8c6229ddd787cad684e0ab9f69b498e98ebc"}, + {file = "kubernetes-31.0.0-py2.py3-none-any.whl", hash = "sha256:bf141e2d380c8520eada8b351f4e319ffee9636328c137aa432bc486ca1200e1"}, + {file = "kubernetes-31.0.0.tar.gz", hash = "sha256:28945de906c8c259c1ebe62703b56a03b714049372196f854105afe4e6d014c0"}, ] [package.dependencies] certifi = ">=14.05.14" +durationpy = ">=0.7" google-auth = ">=1.0.1" oauthlib = ">=3.2.2" python-dateutil = ">=2.5.3" @@ -2536,13 +2565,13 @@ dev = ["bumpver", "isort", "mypy", "pylint", "pytest", "yapf"] [[package]] name = "msgraph-sdk" -version = "1.6.0" +version = "1.8.0" description = "The Microsoft Graph Python SDK" optional = false python-versions = ">=3.8" files = [ - {file = "msgraph_sdk-1.6.0-py3-none-any.whl", hash = "sha256:00dc0037a0a15a00bf04a9fd1340bb14b70dd0bbd5bc881bfde4ec9c0e2b9dbc"}, - {file = "msgraph_sdk-1.6.0.tar.gz", hash = "sha256:6724b186564b5a208e5069adff4a491a5401f36ed6928edcc4b6702b82b44c94"}, + {file = "msgraph_sdk-1.8.0-py3-none-any.whl", hash = "sha256:22a8e4a63f989865228f66a54501bef8105909c7156fe0a079ca9b5296339cc2"}, + {file = "msgraph_sdk-1.8.0.tar.gz", hash = "sha256:1ac84bd47ea288a84f46f6c6d0c89d164ee3453b917615632652344538098314"}, ] [package.dependencies] @@ -2876,40 +2905,53 @@ files = [ [[package]] name = "pandas" -version = "2.2.2" +version = "2.2.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, - {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, - {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, - {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, - {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, - {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"}, - {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, - {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, - {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, - {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, - {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, - {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, - {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, - {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, - {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"}, - {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"}, - {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, ] [package.dependencies] @@ -3184,16 +3226,17 @@ develop = false [package.dependencies] alive-progress = "3.1.5" awsipranges = "0.3.3" -azure-identity = "1.17.1" +azure-identity = "1.18.0" azure-keyvault-keys = "4.9.0" azure-mgmt-applicationinsights = "4.0.0" azure-mgmt-authorization = "4.0.0" azure-mgmt-compute = "33.0.0" -azure-mgmt-containerservice = "31.0.0" -azure-mgmt-cosmosdb = "9.5.1" +azure-mgmt-containerregistry = "10.3.0" +azure-mgmt-containerservice = "32.0.0" +azure-mgmt-cosmosdb = "9.6.0" azure-mgmt-keyvault = "10.3.1" azure-mgmt-monitor = "6.0.2" -azure-mgmt-network = "26.0.0" +azure-mgmt-network = "27.0.0" azure-mgmt-rdbms = "10.1.0" azure-mgmt-resource = "23.1.1" azure-mgmt-security = "7.0.0" @@ -3201,29 +3244,29 @@ azure-mgmt-sql = "3.0.1" azure-mgmt-storage = "21.2.1" azure-mgmt-subscription = "3.1.1" azure-mgmt-web = "7.3.1" -azure-storage-blob = "12.22.0" -boto3 = "1.34.162" -botocore = "1.34.162" +azure-storage-blob = "12.23.1" +boto3 = "1.35.28" +botocore = "1.35.29" colorama = "0.4.6" cryptography = "43.0.1" -dash = "2.18.0" +dash = "2.18.1" dash-bootstrap-components = "1.6.0" detect-secrets = "1.5.0" -google-api-python-client = "2.144.0" +google-api-python-client = "2.147.0" google-auth-httplib2 = ">=0.1,<0.3" jsonschema = "4.23.0" -kubernetes = "30.1.0" +kubernetes = "31.0.0" microsoft-kiota-abstractions = "1.3.3" -msgraph-sdk = "1.6.0" +msgraph-sdk = "1.8.0" numpy = "2.0.2" -pandas = "2.2.2" +pandas = "2.2.3" py-ocsf-models = "0.1.1" pydantic = "1.10.15" python-dateutil = "^2.9.0.post0" -pytz = "2024.1" +pytz = "2024.2" schema = "0.7.7" shodan = "1.31.0" -slack-sdk = "3.32.0" +slack-sdk = "3.33.1" tabulate = "0.9.0" tzlocal = "5.2" @@ -3231,7 +3274,7 @@ tzlocal = "5.2" type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "master" -resolved_reference = "1ef15f0b24bdf07b5ce901f3c7c85d1e7c645246" +resolved_reference = "38e024216cd51e2f24b3210ec9008df7f1e54cac" [[package]] name = "psutil" @@ -3747,13 +3790,13 @@ files = [ [[package]] name = "pytz" -version = "2024.1" +version = "2024.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, - {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] [[package]] @@ -4330,13 +4373,13 @@ files = [ [[package]] name = "slack-sdk" -version = "3.32.0" +version = "3.33.1" description = "The Slack API Platform SDK for Python" optional = false python-versions = ">=3.6" files = [ - {file = "slack_sdk-3.32.0-py2.py3-none-any.whl", hash = "sha256:f35e85f2847e6c25cf7c2d1df206ca0ad75556263fb592457bf03cca68ef64bb"}, - {file = "slack_sdk-3.32.0.tar.gz", hash = "sha256:af8fc4ef1d1cbcecd28d01acf6955a3bb5b13d56f0a43a1b1c7e3b212cc5ec5b"}, + {file = "slack_sdk-3.33.1-py2.py3-none-any.whl", hash = "sha256:ef93beec3ce9c8f64da02fd487598a05ec4bc9c92ceed58f122dbe632691cbe2"}, + {file = "slack_sdk-3.33.1.tar.gz", hash = "sha256:e328bb661d95db5f66b993b1d64288ac7c72201a745b4c7cf8848dafb7b74e40"}, ] [package.extras] diff --git a/src/backend/api/db_utils.py b/src/backend/api/db_utils.py index 774ae0953b..b597a34676 100644 --- a/src/backend/api/db_utils.py +++ b/src/backend/api/db_utils.py @@ -2,7 +2,7 @@ from contextlib import contextmanager from django.conf import settings from django.contrib.auth.models import BaseUserManager -from django.db import models +from django.db import models, transaction, connection from psycopg2 import connect as psycopg2_connect from psycopg2.extensions import new_type, register_type, register_adapter, AsIs @@ -40,6 +40,14 @@ def psycopg_connection(database_alias: str): psycopg2_connection.close() +@contextmanager +def tenant_transaction(tenant_id: str): + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute(f"SELECT set_config('api.tenant_id', '{tenant_id}', TRUE);") + yield cursor + + class CustomUserManager(BaseUserManager): def create_user(self, email, password=None, **extra_fields): if not email: diff --git a/src/backend/api/middleware.py b/src/backend/api/middleware.py index 5d4f02537d..d4e5d5d2c4 100644 --- a/src/backend/api/middleware.py +++ b/src/backend/api/middleware.py @@ -1,22 +1,16 @@ import logging import time -from django.http import HttpRequest - from config.custom_logging import BackendLogger -def extract_tenant_id(request: HttpRequest) -> str | None: - """ - Extract the tenant ID from the request headers. - - Args: - request (HttpRequest): The HTTP request object. - - Returns: - str: The tenant ID if present in the headers, otherwise None. - """ - return request.headers.get("X-Tenant-ID") +def extract_auth_info(request) -> dict: + if not hasattr(request, "auth") or request.auth is not None: + tenant_id = request.auth.get("tenant_id", "N/A") + user_id = request.auth.get("user_id", "N/A") + else: + tenant_id, user_id = "N/A", "N/A" + return {"tenant_id": tenant_id, "user_id": user_id} class APILoggingMiddleware: @@ -38,15 +32,17 @@ class APILoggingMiddleware: response = self.get_response(request) duration = time.time() - request_start_time + auth_info = extract_auth_info(request) self.logger.info( "", extra={ + "user_id": auth_info["user_id"], + "tenant_id": auth_info["tenant_id"], "method": request.method, "path": request.path, "query_params": request.GET.dict(), "status_code": response.status_code, "duration": duration, - "tenant_id": extract_tenant_id(request), }, ) diff --git a/src/backend/api/migrations/0001_initial.py b/src/backend/api/migrations/0001_initial.py index db46623e09..40cb5da16a 100644 --- a/src/backend/api/migrations/0001_initial.py +++ b/src/backend/api/migrations/0001_initial.py @@ -420,6 +420,70 @@ class Migration(migrations.Migration): name="unique_provider_uids", ), ), + migrations.CreateModel( + name="Task", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ( + "task_runner_task", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="task", + related_query_name="task", + to="django_celery_results.taskresult", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "tasks", + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="task", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_task", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddIndex( + model_name="task", + index=models.Index( + fields=["id", "task_runner_task"], + name="tasks_id_trt_id_idx", + ), + ), + migrations.RunSQL( + f""" + ALTER TABLE {TASK_RUNNER_DB_TABLE} ENABLE ROW LEVEL SECURITY; + CREATE POLICY "{DB_PROWLER_USER}_{TASK_RUNNER_DB_TABLE}_select" + ON {TASK_RUNNER_DB_TABLE} + FOR SELECT + TO {DB_PROWLER_USER} + USING ( + task_id::uuid in (SELECT id FROM tasks WHERE tenant_id = (NULLIF(current_setting('{POSTGRES_TENANT_VAR}', true), ''))::uuid) + ); + GRANT SELECT ON TABLE {TASK_RUNNER_DB_TABLE} TO {DB_PROWLER_USER}; + """ + ), # Create and register ScanTriggerEnum type migrations.RunPython( ScanTriggerEnumMigration.create_enum_type, @@ -488,6 +552,17 @@ class Migration(migrations.Migration): to="api.provider", ), ), + ( + "task", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="scans", + related_query_name="scan", + to="api.task", + null=True, + blank=True, + ), + ), ( "tenant", models.ForeignKey( @@ -515,70 +590,6 @@ class Migration(migrations.Migration): name="scans_prov_state_trig_sche_idx", ), ), - migrations.CreateModel( - name="Task", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ("inserted_at", models.DateTimeField(auto_now_add=True)), - ( - "task_runner_task", - models.OneToOneField( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="task", - related_query_name="task", - to="django_celery_results.taskresult", - ), - ), - ( - "tenant", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="api.tenant" - ), - ), - ], - options={ - "db_table": "tasks", - "abstract": False, - }, - ), - migrations.AddConstraint( - model_name="task", - constraint=api.rls.RowLevelSecurityConstraint( - "tenant_id", - name="rls_on_task", - statements=["SELECT", "INSERT", "UPDATE", "DELETE"], - ), - ), - migrations.AddIndex( - model_name="task", - index=models.Index( - fields=["id", "task_runner_task"], - name="tasks_id_trt_id_idx", - ), - ), - migrations.RunSQL( - f""" - ALTER TABLE {TASK_RUNNER_DB_TABLE} ENABLE ROW LEVEL SECURITY; - CREATE POLICY "{DB_PROWLER_USER}_{TASK_RUNNER_DB_TABLE}_select" - ON {TASK_RUNNER_DB_TABLE} - FOR SELECT - TO {DB_PROWLER_USER} - USING ( - task_id::uuid in (SELECT id FROM tasks WHERE tenant_id = (NULLIF(current_setting('{POSTGRES_TENANT_VAR}', true), ''))::uuid) - ); - GRANT SELECT ON TABLE {TASK_RUNNER_DB_TABLE} TO {DB_PROWLER_USER}; - """ - ), # Resources migrations.RunSQL( sql=""" @@ -812,7 +823,7 @@ class Migration(migrations.Migration): constraint=api.rls.RowLevelSecurityConstraint( "tenant_id", name="rls_on_resourcetag", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), migrations.AddConstraint( @@ -827,13 +838,13 @@ class Migration(migrations.Migration): constraint=api.rls.RowLevelSecurityConstraint( "tenant_id", name="rls_on_resourcetagmapping", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), migrations.AddConstraint( model_name="resource", constraint=models.UniqueConstraint( - fields=("tenant_id", "provider_id", "uid"), + fields=("tenant_id", "provider_id", "uid", "region"), name="unique_resources_by_provider", ), ), @@ -842,7 +853,7 @@ class Migration(migrations.Migration): constraint=api.rls.RowLevelSecurityConstraint( "tenant_id", name="rls_on_resource", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), # Create and register ScanTypeEnum type @@ -1084,7 +1095,7 @@ class Migration(migrations.Migration): constraint=api.rls.RowLevelSecurityConstraint( "tenant_id", name="rls_on_resourcefindingmapping", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), PostgresAddDefaultPartition( @@ -1097,7 +1108,7 @@ class Migration(migrations.Migration): "tenant_id", name="rls_on_resource_finding_mappings_default", partition_name="default", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ), migrations.AlterModelOptions( diff --git a/src/backend/api/models.py b/src/backend/api/models.py index 92da9bb533..f6e1c0baf6 100644 --- a/src/backend/api/models.py +++ b/src/backend/api/models.py @@ -216,56 +216,6 @@ class Provider(RowLevelSecurityProtectedModel): ] -class Scan(RowLevelSecurityProtectedModel): - class TriggerChoices(models.TextChoices): - SCHEDULED = "scheduled", _("Scheduled") - MANUAL = "manual", _("Manual") - - id = models.UUIDField(primary_key=True, default=uuid7, editable=False) - name = models.CharField( - blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)] - ) - provider = models.ForeignKey( - Provider, - on_delete=models.CASCADE, - related_name="scans", - related_query_name="scan", - ) - trigger = ScanTriggerEnumField( - choices=TriggerChoices.choices, - ) - state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE) - unique_resource_count = models.IntegerField(default=0) - progress = models.IntegerField(default=0) - scanner_args = models.JSONField(default=dict) - duration = models.IntegerField(null=True, blank=True) - scheduled_at = models.DateTimeField(null=True, blank=True) - inserted_at = models.DateTimeField(auto_now_add=True, editable=False) - updated_at = models.DateTimeField(auto_now=True, editable=False) - started_at = models.DateTimeField(null=True, blank=True) - completed_at = models.DateTimeField(null=True, blank=True) - # TODO: task foreign key - # TODO: mutelist foreign key - - class Meta(RowLevelSecurityProtectedModel.Meta): - db_table = "scans" - - constraints = [ - RowLevelSecurityConstraint( - field="tenant_id", - name="rls_on_%(class)s", - statements=["SELECT", "INSERT", "UPDATE", "DELETE"], - ), - ] - - indexes = [ - models.Index( - fields=["provider", "state", "trigger", "scheduled_at"], - name="scans_prov_state_trig_sche_idx", - ), - ] - - class Task(RowLevelSecurityProtectedModel): id = models.UUIDField(primary_key=True, default=uuid4, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) @@ -297,6 +247,63 @@ class Task(RowLevelSecurityProtectedModel): ] +class Scan(RowLevelSecurityProtectedModel): + class TriggerChoices(models.TextChoices): + SCHEDULED = "scheduled", _("Scheduled") + MANUAL = "manual", _("Manual") + + id = models.UUIDField(primary_key=True, default=uuid7, editable=False) + name = models.CharField( + blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)] + ) + provider = models.ForeignKey( + Provider, + on_delete=models.CASCADE, + related_name="scans", + related_query_name="scan", + ) + task = models.ForeignKey( + Task, + on_delete=models.CASCADE, + related_name="scans", + related_query_name="scan", + null=True, + blank=True, + ) + trigger = ScanTriggerEnumField( + choices=TriggerChoices.choices, + ) + state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE) + unique_resource_count = models.IntegerField(default=0) + progress = models.IntegerField(default=0) + scanner_args = models.JSONField(default=dict) + duration = models.IntegerField(null=True, blank=True) + scheduled_at = models.DateTimeField(null=True, blank=True) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + # TODO: mutelist foreign key + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "scans" + + constraints = [ + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + indexes = [ + models.Index( + fields=["provider", "state", "trigger", "scheduled_at"], + name="scans_prov_state_trig_sche_idx", + ), + ] + + class ResourceTag(RowLevelSecurityProtectedModel): id = models.UUIDField(primary_key=True, default=uuid4, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) @@ -329,7 +336,7 @@ class ResourceTag(RowLevelSecurityProtectedModel): RowLevelSecurityConstraint( field="tenant_id", name="rls_on_%(class)s", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] @@ -409,13 +416,13 @@ class Resource(RowLevelSecurityProtectedModel): constraints = [ models.UniqueConstraint( - fields=("tenant_id", "provider_id", "uid"), + fields=("tenant_id", "provider_id", "uid", "region"), name="unique_resources_by_provider", ), RowLevelSecurityConstraint( field="tenant_id", name="rls_on_%(class)s", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] @@ -444,7 +451,7 @@ class ResourceTagMapping(RowLevelSecurityProtectedModel): RowLevelSecurityConstraint( field="tenant_id", name="rls_on_%(class)s", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] @@ -485,9 +492,11 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): impact_extended = models.TextField(blank=True, null=True) raw_result = models.JSONField(default=dict) + # TODO: review usability tags = models.JSONField(default=dict, null=True, blank=True) check_id = models.CharField(max_length=100, blank=False, null=False) + # TODO: review usability check_metadata = models.JSONField(default=dict, null=False) # Relationships @@ -591,6 +600,6 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected RowLevelSecurityConstraint( field="tenant_id", name="rls_on_%(class)s", - statements=["SELECT"], + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] diff --git a/src/backend/api/specs/v1.yaml b/src/backend/api/specs/v1.yaml index 41947df798..967a7c7aea 100644 --- a/src/backend/api/specs/v1.yaml +++ b/src/backend/api/specs/v1.yaml @@ -651,7 +651,7 @@ paths: content: application/vnd.api+json: schema: - $ref: '#/components/schemas/SerializerMetaclassResponse' + $ref: '#/components/schemas/OpenApiResponseResponse' description: '' /api/v1/providers/{id}/connection: post: @@ -676,7 +676,7 @@ paths: content: application/vnd.api+json: schema: - $ref: '#/components/schemas/SerializerMetaclassResponse' + $ref: '#/components/schemas/OpenApiResponseResponse' description: '' /api/v1/resources: get: @@ -1013,6 +1013,7 @@ paths: - scanner_args - duration - provider + - task - started_at - completed_at - scheduled_at @@ -1181,11 +1182,11 @@ paths: security: - jwtAuth: [] responses: - '201': + '202': content: application/vnd.api+json: schema: - $ref: '#/components/schemas/ScanCreateResponse' + $ref: '#/components/schemas/OpenApiResponseResponse' description: '' /api/v1/scans/{id}: get: @@ -1208,6 +1209,7 @@ paths: - scanner_args - duration - provider + - task - started_at - completed_at - scheduled_at @@ -1267,7 +1269,7 @@ paths: content: application/vnd.api+json: schema: - $ref: '#/components/schemas/SerializerMetaclassResponse' + $ref: '#/components/schemas/ScanUpdateResponse' description: '' /api/v1/tasks: get: @@ -1415,7 +1417,7 @@ paths: content: application/vnd.api+json: schema: - $ref: '#/components/schemas/SerializerMetaclassResponse' + $ref: '#/components/schemas/OpenApiResponseResponse' description: '' /api/v1/tenants: get: @@ -1781,7 +1783,8 @@ paths: /api/v1/tokens/refresh: post: operationId: tokens_refresh_create - description: Refresh an access token by providing a valid refresh token. + description: Refresh an access token by providing a valid refresh token. Former + refresh tokens are invalidated when a new one is issued. summary: Refresh a token tags: - Token @@ -2111,35 +2114,6 @@ paths: description: '' components: schemas: - DelayedTask: - type: object - required: - - type - additionalProperties: false - properties: - type: - allOf: - - $ref: '#/components/schemas/TypeB52Enum' - description: The [type](https://jsonapi.org/format/#document-resource-object-identification) - member is used to describe resource objects that share common attributes - and relationships. - attributes: - type: object - properties: - id: - type: string - state: - type: string - enum: - - available - - scheduled - - executing - - completed - - failed - - cancelled - readOnly: true - required: - - id Finding: type: object required: @@ -2376,6 +2350,13 @@ components: type: string enum: - Membership + OpenApiResponseResponse: + type: object + properties: + data: + $ref: '#/components/schemas/Task' + required: + - data PaginatedFindingList: type: object properties: @@ -2988,33 +2969,7 @@ components: - data description: The identifier of the related object. title: Resource Identifier - required: - - provider - ScanCreate: - type: object - required: - - type - additionalProperties: false - properties: - type: - allOf: - - $ref: '#/components/schemas/Type2bbEnum' - description: The [type](https://jsonapi.org/format/#document-resource-object-identification) - member is used to describe resource objects that share common attributes - and relationships. - attributes: - type: object - properties: - scanner_args: {} - name: - type: string - nullable: true - maxLength: 100 - minLength: 3 - relationships: - type: object - properties: - provider: + task: type: object properties: data: @@ -3026,7 +2981,7 @@ components: type: type: string enum: - - Provider + - Task title: Resource Type Name description: The [type](https://jsonapi.org/format/#document-resource-object-identification) member is used to describe resource objects that share common @@ -3038,6 +2993,7 @@ components: - data description: The identifier of the related object. title: Resource Identifier + nullable: true required: - provider ScanCreateRequest: @@ -3096,13 +3052,6 @@ components: - provider required: - data - ScanCreateResponse: - type: object - properties: - data: - $ref: '#/components/schemas/ScanCreate' - required: - - data ScanResponse: type: object properties: @@ -3110,6 +3059,37 @@ components: $ref: '#/components/schemas/Scan' required: - data + ScanUpdate: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/Type2bbEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: + type: string + format: uuid + attributes: + type: object + properties: + name: + type: string + nullable: true + maxLength: 100 + minLength: 3 + ScanUpdateResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ScanUpdate' + required: + - data SerializerMetaclassResponse: type: object properties: @@ -3121,18 +3101,14 @@ components: type: object required: - type - - id additionalProperties: false properties: type: allOf: - - $ref: '#/components/schemas/TypeB52Enum' + - $ref: '#/components/schemas/TaskTypeEnum' description: The [type](https://jsonapi.org/format/#document-resource-object-identification) member is used to describe resource objects that share common attributes and relationships. - id: - type: string - format: uuid attributes: type: object properties: @@ -3170,6 +3146,10 @@ components: $ref: '#/components/schemas/Task' required: - data + TaskTypeEnum: + type: string + enum: + - Task Tenant: type: object required: @@ -3466,10 +3446,6 @@ components: type: string enum: - User - TypeB52Enum: - type: string - enum: - - Task User: type: object required: diff --git a/src/backend/api/tests/test_middleware.py b/src/backend/api/tests/test_middleware.py index a6c3faeded..1bd8a07351 100644 --- a/src/backend/api/tests/test_middleware.py +++ b/src/backend/api/tests/test_middleware.py @@ -1,33 +1,54 @@ -from unittest.mock import patch, Mock, ANY +from unittest.mock import MagicMock, patch import pytest +from django.http import HttpResponse +from django.test import RequestFactory + +from api.middleware import APILoggingMiddleware @pytest.mark.django_db @patch("logging.getLogger") -def test_api_logger_middleware(mock_get_logger, client): - mock_logger = Mock() - mock_get_logger.side_effect = lambda name: mock_logger if name == "api" else Mock() - request_method = "GET" - request_path = "/this_path_does_not_exist" - tenant_id = "12646005-9067-4d2a-a098-8bb378604362" - api_logger = "api" +def test_api_logging_middleware_logging(mock_logger): + factory = RequestFactory() - response = getattr(client, request_method.lower())( - request_path, headers={"X-Tenant-ID": tenant_id} - ) + request = factory.get("/test-path?param1=value1¶m2=value2") + request.method = "GET" - mock_get_logger.assert_any_call(api_logger) - mock_logger.info.assert_called_once_with( - "", - extra={ - "method": request_method, - "path": request_path, - "query_params": {}, - "status_code": response.status_code, - "duration": ANY, - "tenant_id": tenant_id, - }, - ) + response = HttpResponse() + response.status_code = 200 - assert isinstance(mock_logger.info.call_args[1]["extra"]["duration"], float) + get_response = MagicMock(return_value=response) + + with patch("api.middleware.extract_auth_info") as mock_extract_auth_info: + mock_extract_auth_info.return_value = { + "user_id": "user123", + "tenant_id": "tenant456", + } + + with patch("api.middleware.logging.getLogger") as mock_get_logger: + mock_logger = MagicMock() + mock_get_logger.return_value = mock_logger + + middleware = APILoggingMiddleware(get_response) + + with patch("api.middleware.time.time") as mock_time: + mock_time.side_effect = [1000.0, 1001.0] # Start time and end time + + middleware(request) + + get_response.assert_called_once_with(request) + + mock_extract_auth_info.assert_called_once_with(request) + + expected_extra = { + "user_id": "user123", + "tenant_id": "tenant456", + "method": "GET", + "path": "/test-path", + "query_params": {"param1": "value1", "param2": "value2"}, + "status_code": 200, + "duration": 1.0, + } + + mock_logger.info.assert_called_once_with("", extra=expected_extra) diff --git a/src/backend/api/tests/test_views.py b/src/backend/api/tests/test_views.py index 1720182e97..15e3e38870 100644 --- a/src/backend/api/tests/test_views.py +++ b/src/backend/api/tests/test_views.py @@ -926,13 +926,21 @@ class TestProviderViewSet: ) assert response.status_code == status.HTTP_400_BAD_REQUEST + @patch("api.v1.views.Task.objects.get") @patch("api.v1.views.delete_provider_task.delay") def test_providers_delete( - self, mock_delete_task, authenticated_client, providers_fixture + self, + mock_delete_task, + mock_task_get, + authenticated_client, + providers_fixture, + tasks_fixture, ): + prowler_task = tasks_fixture[0] task_mock = Mock() - task_mock.id = "12345" + task_mock.id = prowler_task.id mock_delete_task.return_value = task_mock + mock_task_get.return_value = prowler_task provider1, *_ = providers_fixture response = authenticated_client.delete( @@ -951,17 +959,22 @@ class TestProviderViewSet: ) assert response.status_code == status.HTTP_404_NOT_FOUND + @patch("api.v1.views.Task.objects.get") @patch("api.v1.views.check_provider_connection_task.delay") def test_providers_connection( self, mock_provider_connection, + mock_task_get, authenticated_client, providers_fixture, + tasks_fixture, ): + prowler_task = tasks_fixture[0] task_mock = Mock() - task_mock.id = "12345" + task_mock.id = prowler_task.id task_mock.status = "PENDING" mock_provider_connection.return_value = task_mock + mock_task_get.return_value = prowler_task provider1, *_ = providers_fixture assert provider1.connected is None @@ -1127,13 +1140,21 @@ class TestScanViewSet: ), ], ) + @patch("api.v1.views.Task.objects.get") + @patch("api.v1.views.perform_scan_task.delay") def test_scans_create_valid( self, + mock_perform_scan_task, + mock_task_get, authenticated_client, scan_json_payload, expected_scanner_args, providers_fixture, + tasks_fixture, ): + prowler_task = tasks_fixture[0] + mock_perform_scan_task.return_value.id = prowler_task.id + mock_task_get.return_value = prowler_task *_, provider5 = providers_fixture # Provider5 has these scanner_args # scanner_args={"key1": "value1", "key2": {"key21": "value21"}} diff --git a/src/backend/api/v1/serializers.py b/src/backend/api/v1/serializers.py index 10888c0b4c..66ffe257eb 100644 --- a/src/backend/api/v1/serializers.py +++ b/src/backend/api/v1/serializers.py @@ -86,7 +86,7 @@ class TokenSerializer(serializers.Serializer): except Exception as e: raise serializers.ValidationError({"detail": str(e)}) - return {"refresh": str(refresh), "access": str(access)} + return {"access": str(access), "refresh": str(refresh)} class TokenRefreshSerializer(serializers.Serializer): @@ -215,7 +215,7 @@ class UserUpdateSerializer(BaseWriteSerializer): # Tasks -class TaskBase(serializers.Serializer): +class TaskBase(serializers.ModelSerializer): state_mapping = { "PENDING": StateChoices.SCHEDULED, "STARTED": StateChoices.EXECUTING, @@ -225,8 +225,9 @@ class TaskBase(serializers.Serializer): "REVOKED": StateChoices.CANCELLED, } - class JSONAPIMeta: - resource_name = "Task" + class Meta: + fields = ["id"] + model = Task def map_state(self, task_result_state): return self.state_mapping.get(task_result_state, StateChoices.AVAILABLE) @@ -244,27 +245,6 @@ class TaskBase(serializers.Serializer): return self.map_state(task_result_state) -class DelayedTaskSerializer(TaskBase): - id = serializers.CharField() - state = serializers.SerializerMethodField(read_only=True) - - class Meta: - fields = [ - "id", - "state", - ] - - @extend_schema_field( - { - "type": "string", - "enum": StateChoices.values, - } - ) - def get_state(self, obj): - task_result_state = obj.status if obj else None - return self.map_state(task_result_state) - - class TaskSerializer(RLSSerializer, TaskBase): state = serializers.SerializerMethodField(read_only=True) metadata = serializers.SerializerMethodField(read_only=True) @@ -444,6 +424,7 @@ class ScanSerializer(RLSSerializer): "scanner_args", "duration", "provider", + "task", "started_at", "completed_at", "scheduled_at", @@ -455,7 +436,7 @@ class ScanCreateSerializer(RLSSerializer, BaseWriteSerializer): class Meta: model = Scan # TODO: add mutelist when implemented - fields = ["provider", "scanner_args", "name"] + fields = ["id", "provider", "scanner_args", "name"] def create(self, validated_data): provider = validated_data.get("provider") @@ -488,9 +469,32 @@ class ScanUpdateSerializer(BaseWriteSerializer): } +class ScanTaskSerializer(RLSSerializer): + trigger = serializers.ChoiceField( + choices=Scan.TriggerChoices.choices, read_only=True + ) + state = StateEnumSerializerField(read_only=True) + + class Meta: + model = Scan + fields = [ + "id", + "name", + "trigger", + "state", + "unique_resource_count", + "progress", + "scanner_args", + "duration", + "started_at", + "completed_at", + "scheduled_at", + ] + + class ResourceTagSerializer(RLSSerializer): """ - Serializer fore the ResourceTag model + Serializer for the ResourceTag model """ class Meta: diff --git a/src/backend/api/v1/views.py b/src/backend/api/v1/views.py index f562d5d858..33c284be74 100644 --- a/src/backend/api/v1/views.py +++ b/src/backend/api/v1/views.py @@ -7,7 +7,12 @@ from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control from drf_spectacular.settings import spectacular_settings from drf_spectacular.utils import OpenApiTypes -from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter +from drf_spectacular.utils import ( + extend_schema, + extend_schema_view, + OpenApiParameter, + OpenApiResponse, +) from drf_spectacular.views import SpectacularAPIView from rest_framework import status, permissions from rest_framework.decorators import action @@ -42,14 +47,17 @@ from api.v1.serializers import ( ProviderUpdateSerializer, TenantSerializer, TaskSerializer, - DelayedTaskSerializer, ScanSerializer, ScanCreateSerializer, ScanUpdateSerializer, ResourceSerializer, FindingSerializer, ) -from tasks.tasks import check_provider_connection_task, delete_provider_task +from tasks.tasks import ( + check_provider_connection_task, + delete_provider_task, + perform_scan_task, +) CACHE_DECORATOR = cache_control( max_age=django_settings.CACHE_MAX_AGE, @@ -390,7 +398,7 @@ class TenantMembersViewSet(BaseTenantViewset): tags=["Provider"], summary="Delete a provider", description="Remove a provider from the system by their ID.", - responses={202: DelayedTaskSerializer}, + responses={202: OpenApiResponse(response=TaskSerializer)}, ), ) @method_decorator(CACHE_DECORATOR, name="list") @@ -420,7 +428,7 @@ class ProviderViewSet(BaseRLSViewSet): elif self.action == "partial_update": return ProviderUpdateSerializer elif self.action in ["connection", "destroy"]: - return DelayedTaskSerializer + return TaskSerializer return super().get_serializer_class() def partial_update(self, request, *args, **kwargs): @@ -443,7 +451,7 @@ class ProviderViewSet(BaseRLSViewSet): summary="Check connection", description="Try to verify connection. For instance, Role & Credentials are set correctly", request=None, - responses={202: DelayedTaskSerializer}, + responses={202: OpenApiResponse(response=TaskSerializer)}, ) @action(detail=True, methods=["post"], url_name="connection") def connection(self, request, pk=None): @@ -451,24 +459,30 @@ class ProviderViewSet(BaseRLSViewSet): task = check_provider_connection_task.delay( provider_id=pk, tenant_id=request.tenant_id ) - serializer = DelayedTaskSerializer(task) + prowler_task = Task.objects.get(id=task.id) + serializer = TaskSerializer(prowler_task) return Response( data=serializer.data, status=status.HTTP_202_ACCEPTED, headers={ - "Content-Location": reverse("task-detail", kwargs={"pk": task.id}) + "Content-Location": reverse( + "task-detail", kwargs={"pk": prowler_task.id} + ) }, ) def destroy(self, request, *args, pk=None, **kwargs): get_object_or_404(Provider, pk=pk) task = delete_provider_task.delay(provider_id=pk, tenant_id=request.tenant_id) - serializer = DelayedTaskSerializer(task) + prowler_task = Task.objects.get(id=task.id) + serializer = TaskSerializer(prowler_task) return Response( data=serializer.data, status=status.HTTP_202_ACCEPTED, headers={ - "Content-Location": reverse("task-detail", kwargs={"pk": task.id}) + "Content-Location": reverse( + "task-detail", kwargs={"pk": prowler_task.id} + ) }, ) @@ -482,6 +496,10 @@ class ProviderViewSet(BaseRLSViewSet): summary="Retrieve data from a specific scan", description="Fetch detailed information about a specific scan by its ID.", ), + partial_update=extend_schema( + summary="Partially update a scan", + description="Update certain fields of an existing scan without affecting other fields.", + ), create=extend_schema( summary="Trigger a manual scan", description=( @@ -491,12 +509,8 @@ class ProviderViewSet(BaseRLSViewSet): "merged with the provider's defaults. This means that your provided settings will override " "the defaults only where they conflict, while the rest of the default settings will remain intact." ), - ), - partial_update=extend_schema( - summary="Partially update a scan", - description="Update certain fields of an existing scan without affecting other fields.", - request=ScanUpdateSerializer, - responses={200: ScanSerializer}, + request=ScanCreateSerializer, + responses={202: OpenApiResponse(response=TaskSerializer)}, ), ) @method_decorator(CACHE_DECORATOR, name="list") @@ -521,25 +535,13 @@ class ScanViewSet(BaseRLSViewSet): def get_serializer_class(self): if self.action == "create": + if hasattr(self, "response_serializer_class"): + return self.response_serializer_class return ScanCreateSerializer elif self.action == "partial_update": return ScanUpdateSerializer return super().get_serializer_class() - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - scan = serializer.save() - - # TODO: Run scan through task and return task info here - return Response( - data=serializer.data, - status=status.HTTP_202_ACCEPTED, - headers={ - "Content-Location": reverse("scan-detail", kwargs={"pk": scan.id}) - }, - ) - def partial_update(self, request, *args, **kwargs): instance = self.get_object() serializer = self.get_serializer( @@ -555,6 +557,35 @@ class ScanViewSet(BaseRLSViewSet): ) return Response(data=read_serializer.data, status=status.HTTP_200_OK) + def create(self, request, *args, **kwargs): + input_serializer = self.get_serializer(data=request.data) + input_serializer.is_valid(raise_exception=True) + scan = input_serializer.save() + + task = perform_scan_task.delay( + tenant_id=request.tenant_id, + scan_id=str(scan.id), + provider_id=str(scan.provider_id), + checks_to_execute=scan.scanner_args.get("checks_to_execute", []), + ) + + scan.task_id = task.id + scan.save(update_fields=["task_id"]) + + prowler_task = Task.objects.get(id=task.id) + self.response_serializer_class = TaskSerializer + output_serializer = self.get_serializer(prowler_task) + + return Response( + data=output_serializer.data, + status=status.HTTP_202_ACCEPTED, + headers={ + "Content-Location": reverse( + "task-detail", kwargs={"pk": prowler_task.id} + ) + }, + ) + @extend_schema_view( list=extend_schema( @@ -569,7 +600,7 @@ class ScanViewSet(BaseRLSViewSet): tags=["Task"], summary="Revoke a task", description="Try to revoke a task using its ID. Only tasks that are not yet in progress can be revoked.", - responses={202: DelayedTaskSerializer}, + responses={202: OpenApiResponse(response=TaskSerializer)}, ), ) class TaskViewSet(BaseRLSViewSet): @@ -602,7 +633,8 @@ class TaskViewSet(BaseRLSViewSet): task_instance = AsyncResult(pk) task_instance.revoke() - serializer = DelayedTaskSerializer(task_instance) + task.refresh_from_db() + serializer = TaskSerializer(task) return Response( data=serializer.data, status=status.HTTP_202_ACCEPTED, diff --git a/src/backend/config/custom_logging.py b/src/backend/config/custom_logging.py index 864c3b63ed..04ef403b1d 100644 --- a/src/backend/config/custom_logging.py +++ b/src/backend/config/custom_logging.py @@ -45,6 +45,10 @@ class NDJSONFormatter(logging.Formatter): } # Add REST API extra fields + if hasattr(record, "user_id"): + log_record["user_id"] = record.user_id + if hasattr(record, "tenant_id"): + log_record["tenant_id"] = record.tenant_id if hasattr(record, "method"): log_record["method"] = record.method if hasattr(record, "path"): @@ -55,8 +59,6 @@ class NDJSONFormatter(logging.Formatter): log_record["duration"] = record.duration if hasattr(record, "status_code"): log_record["status_code"] = record.status_code - if hasattr(record, "tenant_id"): - log_record["tenant_id"] = record.tenant_id if record.exc_info: log_record["exc_info"] = self.formatException(record.exc_info) @@ -82,13 +84,15 @@ class HumanReadableFormatter(logging.Formatter): f"function={record.funcName}", f"process={record.process}", f"thread={record.thread}", - f"transaction-id={record.transaction_id if hasattr(record, "transaction_id") else None}]", + f"transaction-id={record.transaction_id if hasattr(record, 'transaction_id') else None}]", f"{record.getMessage()}", ] # Add REST API extra fields + if hasattr(record, "user_id"): + log_components.append(f"({record.user_id})") if hasattr(record, "tenant_id"): - log_components.append(f"({record.tenant_id})") + log_components.append(f"[{record.tenant_id}]") if hasattr(record, "method"): log_components.append(f'"{record.method} {record.path}"') if hasattr(record, "query_params"): diff --git a/src/backend/tasks/jobs/connection.py b/src/backend/tasks/jobs/connection.py index e00282aef6..feb189c61c 100644 --- a/src/backend/tasks/jobs/connection.py +++ b/src/backend/tasks/jobs/connection.py @@ -2,6 +2,9 @@ from datetime import datetime, timezone from celery.utils.log import get_task_logger from prowler.providers.aws.aws_provider import AwsProvider +from prowler.providers.azure.azure_provider import AzureProvider +from prowler.providers.gcp.gcp_provider import GcpProvider +from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider from api.models import Provider @@ -26,20 +29,26 @@ def check_provider_connection(provider_id: str): """ provider_instance = Provider.objects.get(pk=provider_id) match provider_instance.provider: + # TODO Refactor when proper credentials are implemented case Provider.ProviderChoices.AWS.value: - # TODO Refactor when proper credentials are implemented - try: - connection_result = AwsProvider.test_connection( - raise_on_exception=False - ) - # TODO: Improve this exception handling when SDK exceptions are implemented - except Exception as e: - logger.warning(str(e)) - raise e + prowler_provider = AwsProvider + case Provider.ProviderChoices.GCP.value: + prowler_provider = GcpProvider + case Provider.ProviderChoices.AZURE.value: + prowler_provider = AzureProvider + case Provider.ProviderChoices.KUBERNETES.value: + prowler_provider = KubernetesProvider case _: raise ValueError( f"Provider type {provider_instance.provider} not supported" ) + try: + connection_result = prowler_provider.test_connection(raise_on_exception=False) + except Exception as e: + logger.warning( + f"Unexpected exception checking {provider_instance.provider} provider connection: {str(e)}" + ) + raise e provider_instance.connected = connection_result.is_connected provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc) diff --git a/src/backend/tasks/jobs/scan.py b/src/backend/tasks/jobs/scan.py new file mode 100644 index 0000000000..1a41734d92 --- /dev/null +++ b/src/backend/tasks/jobs/scan.py @@ -0,0 +1,146 @@ +import json +import time +from datetime import datetime, timezone + +from celery.utils.log import get_task_logger +from prowler.lib.outputs.finding import Finding as ProwlerFinding +from prowler.lib.scan.scan import Scan as ProwlerScan +from prowler.providers.aws.aws_provider import AwsProvider +from prowler.providers.azure.azure_provider import AzureProvider +from prowler.providers.gcp.gcp_provider import GcpProvider +from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider + +from api.db_utils import tenant_transaction +from api.models import ( + Provider, + Scan, + Finding, + Resource, + ResourceTag, + StatusChoices as FindingStatus, + StateChoices, +) +from api.v1.serializers import ScanTaskSerializer +from config.celery import TaskTimeoutError + +logger = get_task_logger(__name__) + + +def store_finding( + finding: ProwlerFinding, + tenant_id: str, + scan_instance: Scan, + resource_instance: Resource, +) -> Finding: + with tenant_transaction(tenant_id): + finding_instance = Finding.objects.create( + tenant_id=tenant_id, + delta=Finding.DeltaChoices.NEW, + status=FindingStatus[finding.status.value], + status_extended=finding.status_extended, + severity=finding.severity.value, + impact=finding.severity.value, + raw_result=json.loads(finding.json()), + check_id=finding.check_id, + scan=scan_instance, + ) + finding_instance.add_resources([resource_instance]) + return finding_instance + + +def store_resources( + finding: ProwlerFinding, tenant_id: str, provider_instance: Provider +) -> tuple[Resource, tuple[str, str]]: + with tenant_transaction(tenant_id): + resource_instance, _ = Resource.objects.get_or_create( + tenant_id=tenant_id, + provider=provider_instance, + uid=finding.resource_uid, + region=finding.region, + service=finding.service_name, + type=finding.resource_type, + ) + with tenant_transaction(tenant_id): + tags = [ + ResourceTag.objects.get_or_create( + tenant_id=tenant_id, key=key, value=value + )[0] + for key, value in finding.resource_tags.items() + ] + resource_instance.upsert_or_delete_tags(tags=tags) + return resource_instance, (resource_instance.uid, resource_instance.region) + + +def perform_prowler_scan( + tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None +): + with tenant_transaction(tenant_id): + exception = None + provider_instance = Provider.objects.get(pk=provider_id) + start_time = time.time() + unique_resources = set() + # Prevent race conditions + while not Scan.objects.filter(id=scan_id).exists(): + if time.time() - start_time > 10: + raise TaskTimeoutError( + f"Could not find scan with given id {scan_id} within 10 seconds" + ) + time.sleep(0.1) + scan_instance = Scan.objects.get(pk=scan_id) + scan_instance.state = StateChoices.EXECUTING + scan_instance.started_at = datetime.now(tz=timezone.utc) + scan_instance.save() + try: + match provider_instance.provider: + case Provider.ProviderChoices.AWS.value: + prowler_provider = AwsProvider + case Provider.ProviderChoices.GCP.value: + prowler_provider = GcpProvider + case Provider.ProviderChoices.AZURE.value: + prowler_provider = AzureProvider + case Provider.ProviderChoices.KUBERNETES.value: + prowler_provider = KubernetesProvider + case _: + raise ValueError( + f"Provider type {provider_instance.provider} not supported" + ) + with tenant_transaction(tenant_id): + connection_status = prowler_provider.test_connection( + raise_on_exception=False + ) + provider_instance.connected = connection_status.is_connected + provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc) + provider_instance.save() + + if connection_status.is_connected is False: + raise ValueError(f"Provider {provider_instance.provider} is not connected") + + prowler_scan = ProwlerScan( + provider=prowler_provider(), checks_to_execute=checks_to_execute or [] + ) + for progress, findings in prowler_scan.scan(): + for finding in findings: + resource_instance, resource_uid_tuple = store_resources( + finding, tenant_id, provider_instance + ) + store_finding(finding, tenant_id, scan_instance, resource_instance) + unique_resources.add(resource_uid_tuple) + with tenant_transaction(tenant_id): + scan_instance.progress = progress + scan_instance.save() + + scan_instance.state = StateChoices.COMPLETED + except Exception as e: + logger.error(f"Error performing scan {scan_id}: {e}") + exception = e + scan_instance.state = StateChoices.FAILED + finally: + with tenant_transaction(tenant_id): + scan_instance.duration = time.time() - start_time + scan_instance.completed_at = datetime.now(tz=timezone.utc) + scan_instance.unique_resource_count = len(unique_resources) + scan_instance.save() + if exception is not None: + raise exception + serializer = ScanTaskSerializer(instance=scan_instance) + return serializer.data diff --git a/src/backend/tasks/tasks.py b/src/backend/tasks/tasks.py index 8fb8f1a199..c848390e76 100644 --- a/src/backend/tasks/tasks.py +++ b/src/backend/tasks/tasks.py @@ -5,6 +5,7 @@ from api.models import Provider from config.celery import RLSTask from tasks.jobs.connection import check_provider_connection from tasks.jobs.deletion import delete_instance +from tasks.jobs.scan import perform_prowler_scan @shared_task(base=RLSTask, name="provider-connection-check") @@ -40,3 +41,31 @@ def delete_provider_task(provider_id: str): including related models if cascading deletes were triggered. """ return delete_instance(model=Provider, pk=provider_id) + + +@shared_task(base=RLSTask, name="scan-perform") +def perform_scan_task( + tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None +): + """ + Task to perform a Prowler scan on a given provider. + + This task runs a Prowler scan on the provider identified by `provider_id` + under the tenant identified by `tenant_id`. The scan will use the `scan_id` + for tracking purposes. + + Args: + tenant_id (str): The tenant ID under which the scan is being performed. + scan_id (str): The ID of the scan to be performed. + provider_id (str): The primary key of the Provider instance to scan. + checks_to_execute (list[str], optional): A list of specific checks to perform during the scan. Defaults to None. + + Returns: + dict: The result of the scan execution, typically including the status and results of the performed checks. + """ + return perform_prowler_scan( + tenant_id=tenant_id, + scan_id=scan_id, + provider_id=provider_id, + checks_to_execute=checks_to_execute, + ) diff --git a/src/backend/tasks/tests/test_scan.py b/src/backend/tasks/tests/test_scan.py new file mode 100644 index 0000000000..bf0d3d4dea --- /dev/null +++ b/src/backend/tasks/tests/test_scan.py @@ -0,0 +1,111 @@ +import json +from unittest.mock import patch, MagicMock, PropertyMock + +import pytest + +from api.models import StateChoices, StatusChoices, Severity, Finding, Resource +from tasks.jobs.scan import perform_prowler_scan + + +@pytest.mark.django_db +class TestPerformScan: + @patch("tasks.jobs.scan.ProwlerScan") + @patch("tasks.jobs.scan.AwsProvider") + @patch("api.db_utils.tenant_transaction") + def test_perform_prowler_scan_success( + self, + mock_tenant_transaction, + mock_aws_provider, + mock_prowler_scan, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + assert len(Finding.objects.all()) == 0 + assert len(Resource.objects.all()) == 0 + + tenant, _ = tenants_fixture + scan, *_ = scans_fixture + provider, *_ = providers_fixture + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + checks_to_execute = ["check1", "check2"] + + connection_status_mock = MagicMock() + type(connection_status_mock).is_connected = PropertyMock(return_value=True) + mock_aws_provider.test_connection.return_value = connection_status_mock + mock_aws_provider_instance = mock_aws_provider.return_value + mock_aws_provider_instance.test_connection.return_value = connection_status_mock + + finding = MagicMock() + finding.status = StatusChoices.PASS + finding.status_extended = "test status extended" + finding.severity = Severity.medium + finding.check_id = "check1" + finding.json.return_value = '{"key": "value"}' + finding.resource_uid = "resource_uid" + finding.region = "region" + finding.service_name = "service_name" + finding.resource_type = "resource_type" + mock_prowler_scan.return_value.scan.return_value = [(100, [finding])] + + perform_prowler_scan(tenant_id, scan_id, provider_id, checks_to_execute) + + scan.refresh_from_db() + scan_finding = Finding.objects.get(scan=scan) + scan_resource = Resource.objects.get(provider=provider) + + assert scan.tenant == tenant + assert scan.provider == provider + assert scan.state == StateChoices.COMPLETED + assert scan.completed_at is not None + assert scan.duration is not None + assert scan.started_at is not None + assert scan.unique_resource_count == 1 + assert scan.progress == 100 + assert scan_finding.status == finding.status + assert scan_finding.status_extended == finding.status_extended + assert scan_finding.severity == finding.severity + assert scan_finding.check_id == finding.check_id + assert scan_finding.raw_result == json.loads(finding.json()) + + assert scan_resource.tenant == tenant + assert scan_resource.uid == finding.resource_uid + assert scan_resource.region == finding.region + assert scan_resource.service == finding.service_name + assert scan_resource.type == finding.resource_type + + @patch("tasks.jobs.scan.ProwlerScan") + @patch("tasks.jobs.scan.AwsProvider") + @patch("api.db_utils.tenant_transaction") + def test_perform_prowler_scan_no_connection( + self, + mock_tenant_transaction, + mock_aws_provider, + mock_prowler_scan, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + tenant, _ = tenants_fixture + scan, *_ = scans_fixture + provider, *_ = providers_fixture + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + provider_id = str(provider.id) + checks_to_execute = ["check1", "check2"] + + connection_status_mock = MagicMock() + type(connection_status_mock).is_connected = PropertyMock(return_value=False) + mock_aws_provider.test_connection.return_value = connection_status_mock + mock_aws_provider_instance = mock_aws_provider.return_value + mock_aws_provider_instance.test_connection.return_value = connection_status_mock + + with pytest.raises(ValueError): + perform_prowler_scan(tenant_id, scan_id, provider_id, checks_to_execute) + + scan.refresh_from_db() + assert scan.state == StateChoices.FAILED