ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-07-23 04:21:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(Scan, Resource, Finding): PRWLR-4015 Implement on demand scans and findings/resources in database (#48)

Browse Source 
* feat(Users): PRWLR-4718 make user email the default login username

* feat(Token): PRWLR-4718 add serializers, views and urls for access and refresh tokens

* feat(Token): PRWLR-4718 add first membership tenant in token if not present in json body

* feat(Users): PRWLR-4718 add company_name to model

* feat(Users): PRWLR-4718 create tenant and membership when creating new user

* fix(BaseView): PRWLR-4718 add tenant_id to serializer context

* fix(Tests): PRWLR-4718 use authorization with unit tests

* fix(Views): PRWLR-4718 fix tenant retrieval from request

* fix(Tests): PRWLR-4718 fix tests

* fix(Fixtures): PRWLR-4718 fix tenant memberships ordering

* chore(Tokens): PRWLR-4718 update token url

* chore(Spec): PRWLR-4718 update API spec

* feat(Tokens): PRWLR-4718 enable token refresh blacklisting

* feat(Tokens): PRWLR-4718 implement RS256 algorithm and dev valid keys

* chore(env): PRWLR-4718 update .env.example

* chore(Deps): PRWLR-4015 update prowler dep

* fix(Resources, Findings): PRWLR-4015 fix permission issues on models and migrations

* feat(Scans, Tasks, Resources, Findings): PRWLR-4015 perform whole flow when executing scans

* fix(Settings): PRWLR-4015 fix devel JWT settings

* chore(Scans, Tasks): PRWLR-4015 improve docs and responses format

* test(Scan, Provider): PRWLR-4015 fix unit tests

* chore(Environment): PRWLR-4015 fix .env.example values

* test(Scan): PRWLR-4015 add unit tests for scan task

* chore(Task): PRWLR-4015 give proper format to task result on scan perform

* feat(Provider, Scan): PRWLR-4015 add all provider types to connection check and scan

* fix(Logging): PRWLR-4015 fix API logger middleware to include tenant and user IDs
This commit is contained in:
Víctor Fernández Poyatos
2024-10-03 18:36:42 +02:00
committed by GitHub
parent 54bb034cac
commit ded28baa2f
16 changed files with 836 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3 -2
View File
@@ -11,10 +11,11 @@ DJANGO_LOGGING_FORMATTER=[ndjson|human_readable]
# applies to both Django and Celery Workers
DJANGO_LOGGING_LEVEL=INFO
DJANGO_WORKERS=4 # Defaults to the maximum available based on CPU cores if not set.
DJANGO_TOKEN_SIGNING_KEY=""
DJANGO_TOKEN_VERIFYING_KEY=""
# Token lifetime is in minutes
DJANGO_ACCESS_TOKEN_LIFETIME=30
DJANGO_REFRESH_TOKEN_LIFETIME=1440
DJANGO_TOKEN_SIGNING_KEY=S3cret
DJANGO_CACHE_MAX_AGE=3600
DJANGO_STALE_WHILE_REVALIDATE=60
@@ -28,7 +29,7 @@ POSTGRES_USER=prowler_user
POSTGRES_PASSWORD=S3cret
POSTGRES_DB=prowler_db
# valkey settings.
# Valkey settings
# If running django and celery on host, use localhost, else use 'valkey'
VALKEY_HOST=[localhost|valkey]
VALKEY_PORT=6379
poetry.lock
Generated
+137 -94
View File
@@ -289,13 +289,13 @@ files = [
[[package]]
name = "azure-core"
version = "1.30.2"
version = "1.31.0"
description = "Microsoft Azure Core Library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "azure-core-1.30.2.tar.gz", hash = "sha256:a14dc210efcd608821aa472d9fb8e8d035d29b68993819147bc290a8ac224472"},
{file = "azure_core-1.30.2-py3-none-any.whl", hash = "sha256:cf019c1ca832e96274ae85abd3d9f752397194d9fea3b41487290562ac8abe4a"},
{file = "azure_core-1.31.0-py3-none-any.whl", hash = "sha256:22954de3777e0250029360ef31d80448ef1be13b80a459bff80ba7073379e2cd"},
{file = "azure_core-1.31.0.tar.gz", hash = "sha256:656a0dd61e1869b1506b7c6a3b31d62f15984b1a573d6326f6aa2f3e4123284b"},
]
[package.dependencies]
@@ -308,20 +308,20 @@ aio = ["aiohttp (>=3.0)"]
[[package]]
name = "azure-identity"
version = "1.17.1"
version = "1.18.0"
description = "Microsoft Azure Identity Library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "azure-identity-1.17.1.tar.gz", hash = "sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea"},
{file = "azure_identity-1.17.1-py3-none-any.whl", hash = "sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382"},
{file = "azure_identity-1.18.0-py3-none-any.whl", hash = "sha256:bccf6106245b49ff41d0c4cd7b72851c5a2ba3a32cef7589da246f5727f26f02"},
{file = "azure_identity-1.18.0.tar.gz", hash = "sha256:f567579a65d8932fa913c76eddf3305101a15e5727a5e4aa5df649a0f553d4c3"},
]
[package.dependencies]
azure-core = ">=1.23.0"
azure-core = ">=1.31.0"
cryptography = ">=2.5"
msal = ">=1.24.0"
msal-extensions = ">=0.3.0"
msal = ">=1.30.0"
msal-extensions = ">=1.2.0"
typing-extensions = ">=4.0.0"
[[package]]
@@ -390,15 +390,31 @@ azure-mgmt-core = ">=1.3.2"
isodate = ">=0.6.1"
typing-extensions = ">=4.6.0"
[[package]]
name = "azure-mgmt-containerregistry"
version = "10.3.0"
description = "Microsoft Azure Container Registry Client Library for Python"
optional = false
python-versions = ">=3.7"
files = [
{file = "azure-mgmt-containerregistry-10.3.0.tar.gz", hash = "sha256:ae21651855dfb19c42d91d6b3a965c6c611e23f8bc4bf7138835e652d2f918e3"},
{file = "azure_mgmt_containerregistry-10.3.0-py3-none-any.whl", hash = "sha256:851e1c57f9bc4a3589c6b21fb627c11fd6cbb57a0388b7dfccd530ba3160805f"},
]
[package.dependencies]
azure-common = ">=1.1,<2.0"
azure-mgmt-core = ">=1.3.2,<2.0.0"
isodate = ">=0.6.1,<1.0.0"
[[package]]
name = "azure-mgmt-containerservice"
version = "31.0.0"
version = "32.0.0"
description = "Microsoft Azure Container Service Management Client Library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "azure-mgmt-containerservice-31.0.0.tar.gz", hash = "sha256:134358d7f88c4d29b4009f91d7619861e1fad5dbea5e147402dd61ad96b5624a"},
{file = "azure_mgmt_containerservice-31.0.0-py3-none-any.whl", hash = "sha256:75c6dbbaf27ed6c8251e34b86fefcfb001fe096c69df9ce93c163de71a2b5275"},
{file = "azure_mgmt_containerservice-32.0.0-py3-none-any.whl", hash = "sha256:66ff0a2651122207db0d5492e86550561f9d81509379abe7d401ed4805a1c66b"},
{file = "azure_mgmt_containerservice-32.0.0.tar.gz", hash = "sha256:ccb587479d8a93ec78f7162590adc82e4fba4de76e4adc3419f33ad225efb292"},
]
[package.dependencies]
@@ -423,19 +439,20 @@ azure-core = ">=1.26.2,<2.0.0"
[[package]]
name = "azure-mgmt-cosmosdb"
version = "9.5.1"
version = "9.6.0"
description = "Microsoft Azure Cosmos DB Management Client Library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "azure-mgmt-cosmosdb-9.5.1.tar.gz", hash = "sha256:4e55d3973f11cf02cf7a8055aee86615233d7cbe3bb227a07c176bac8bc7ef02"},
{file = "azure_mgmt_cosmosdb-9.5.1-py3-none-any.whl", hash = "sha256:860cad5583d63936e5f9477f7878a256c4dee039169cdea554da851f0762f0c7"},
{file = "azure_mgmt_cosmosdb-9.6.0-py3-none-any.whl", hash = "sha256:02b4108867de58e0b89a206ee7b7588b439e1f6fef2377ce1979b803a0d02d5a"},
{file = "azure_mgmt_cosmosdb-9.6.0.tar.gz", hash = "sha256:667c7d8a8f542b0e7972e63274af536ad985187e24a6cc2e3c8eef35560881fc"},
]
[package.dependencies]
azure-common = ">=1.1"
azure-mgmt-core = ">=1.3.2"
isodate = ">=0.6.1"
typing-extensions = ">=4.6.0"
[[package]]
name = "azure-mgmt-keyvault"
@@ -472,19 +489,20 @@ isodate = ">=0.6.1,<1.0.0"
[[package]]
name = "azure-mgmt-network"
version = "26.0.0"
version = "27.0.0"
description = "Microsoft Azure Network Management Client Library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "azure-mgmt-network-26.0.0.tar.gz", hash = "sha256:4de676184195053fdb106a6ea1042a894e70c731a6d3c8a633d52f5229f4ee1b"},
{file = "azure_mgmt_network-26.0.0-py3-none-any.whl", hash = "sha256:15d6a77d7429bdcc76f8482158845d42f9d9434529e1ad2b6778269cc14627e9"},
{file = "azure_mgmt_network-27.0.0-py3-none-any.whl", hash = "sha256:67295fda351fdae79d14591487009e911346adea9578bb1b04636a15e3472598"},
{file = "azure_mgmt_network-27.0.0.tar.gz", hash = "sha256:5c1c61d8bb13ad40f788a26fd7569c1d9d60eb2e4cb19c2a1b5d9c02ae862316"},
]
[package.dependencies]
azure-common = ">=1.1"
azure-mgmt-core = ">=1.3.2"
isodate = ">=0.6.1"
typing-extensions = ">=4.6.0"
[[package]]
name = "azure-mgmt-rdbms"
@@ -601,23 +619,23 @@ typing-extensions = ">=4.6.0"
[[package]]
name = "azure-storage-blob"
version = "12.22.0"
version = "12.23.1"
description = "Microsoft Azure Blob Storage Client Library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "azure-storage-blob-12.22.0.tar.gz", hash = "sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e"},
{file = "azure_storage_blob-12.22.0-py3-none-any.whl", hash = "sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8"},
{file = "azure_storage_blob-12.23.1-py3-none-any.whl", hash = "sha256:1c2238aa841d1545f42714a5017c010366137a44a0605da2d45f770174bfc6b4"},
{file = "azure_storage_blob-12.23.1.tar.gz", hash = "sha256:a587e54d4e39d2a27bd75109db164ffa2058fe194061e5446c5a89bca918272f"},
]
[package.dependencies]
azure-core = ">=1.28.0"
azure-core = ">=1.30.0"
cryptography = ">=2.1.4"
isodate = ">=0.6.1"
typing-extensions = ">=4.6.0"
[package.extras]
aio = ["azure-core[aio] (>=1.28.0)"]
aio = ["azure-core[aio] (>=1.30.0)"]
[[package]]
name = "bandit"
@@ -667,17 +685,17 @@ files = [
[[package]]
name = "boto3"
version = "1.34.162"
version = "1.35.28"
description = "The AWS SDK for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "boto3-1.34.162-py3-none-any.whl", hash = "sha256:d6f6096bdab35a0c0deff469563b87d184a28df7689790f7fe7be98502b7c590"},
{file = "boto3-1.34.162.tar.gz", hash = "sha256:873f8f5d2f6f85f1018cbb0535b03cceddc7b655b61f66a0a56995238804f41f"},
{file = "boto3-1.35.28-py3-none-any.whl", hash = "sha256:dc088b86a14f17d3cd2e96915c6ccfd31bce640dfe9180df579ed311bc6bf0fc"},
{file = "boto3-1.35.28.tar.gz", hash = "sha256:8960fc458b9ba3c8a9890a607c31cee375db821f39aefaec9ff638248e81644a"},
]
[package.dependencies]
botocore = ">=1.34.162,<1.35.0"
botocore = ">=1.35.28,<1.36.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.10.0,<0.11.0"
@@ -686,13 +704,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.34.162"
version = "1.35.29"
description = "Low-level, data-driven core of boto 3."
optional = false
python-versions = ">=3.8"
files = [
{file = "botocore-1.34.162-py3-none-any.whl", hash = "sha256:2d918b02db88d27a75b48275e6fb2506e9adaaddbec1ffa6a8a0898b34e769be"},
{file = "botocore-1.34.162.tar.gz", hash = "sha256:adc23be4fb99ad31961236342b7cbf3c0bfc62532cd02852196032e8c0d682f3"},
{file = "botocore-1.35.29-py3-none-any.whl", hash = "sha256:f8e3ae0d84214eff3fb69cb4dc51cea6c43d3bde82027a94d00c52b941d6c3d5"},
{file = "botocore-1.35.29.tar.gz", hash = "sha256:4ed28ab03675bb008a290c452c5ddd7aaa5d4e3fa1912aadbdf93057ee84362b"},
]
[package.dependencies]
@@ -701,7 +719,7 @@ python-dateutil = ">=2.1,<3.0.0"
urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}
[package.extras]
crt = ["awscrt (==0.21.2)"]
crt = ["awscrt (==0.21.5)"]
[[package]]
name = "cachetools"
@@ -1149,13 +1167,13 @@ test-randomorder = ["pytest-randomly"]
[[package]]
name = "dash"
version = "2.18.0"
version = "2.18.1"
description = "A Python framework for building reactive web-apps. Developed by Plotly."
optional = false
python-versions = ">=3.8"
files = [
{file = "dash-2.18.0-py3-none-any.whl", hash = "sha256:2d37b479be20bef1a184d35be4a654e4da131da6538d563fc813be3e28b90cbc"},
{file = "dash-2.18.0.tar.gz", hash = "sha256:9be6ea7562d40bb70211df8cbfb23611cc2ca9ae63cfca9af8f96679989adc40"},
{file = "dash-2.18.1-py3-none-any.whl", hash = "sha256:07c4513bb5f79a4b936847a0b49afc21dbd4b001ff77ea78d4d836043e211a07"},
{file = "dash-2.18.1.tar.gz", hash = "sha256:ffdf89690d734f6851ef1cb344222826ffb11ad2214ab9172668bf8aadd75d12"},
]
[package.dependencies]
@@ -1618,6 +1636,16 @@ djangorestframework-jsonapi = ">=6.0.0"
drf-extensions = ">=0.7.1"
drf-spectacular = ">=0.25.0"
[[package]]
name = "durationpy"
version = "0.8"
description = "Module for converting between datetime.timedelta and Go's Duration strings."
optional = false
python-versions = "*"
files = [
{file = "durationpy-0.8.tar.gz", hash = "sha256:e90941562e260bdb0e2244cd9eee9be5b3aa0efc0557fca754f36d18fe080f9d"},
]
[[package]]
name = "email-validator"
version = "2.1.1"
@@ -1810,13 +1838,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
[[package]]
name = "google-api-python-client"
version = "2.144.0"
version = "2.147.0"
description = "Google API Client Library for Python"
optional = false
python-versions = ">=3.7"
files = [
{file = "google_api_python_client-2.144.0-py2.py3-none-any.whl", hash = "sha256:f9c333ac4454a012adca90c297f9a22611a8953f3aae5481f90b3a56b9bdd413"},
{file = "google_api_python_client-2.144.0.tar.gz", hash = "sha256:fe00851b257157bca600e1692ed8a54762c4a5c7d9eb7f6d4822059424b0d0a9"},
{file = "google_api_python_client-2.147.0-py2.py3-none-any.whl", hash = "sha256:c6ecfa193c695baa41e84562d8f8f244fcd164419eca3fc9fd7565646668f9b2"},
{file = "google_api_python_client-2.147.0.tar.gz", hash = "sha256:e864c2cf61d34c00f05278b8bdb72b93b6fa34f0de9ead51d20435f3b65f91be"},
]
[package.dependencies]
@@ -2212,17 +2240,18 @@ zookeeper = ["kazoo (>=2.8.0)"]
[[package]]
name = "kubernetes"
version = "30.1.0"
version = "31.0.0"
description = "Kubernetes python client"
optional = false
python-versions = ">=3.6"
files = [
{file = "kubernetes-30.1.0-py2.py3-none-any.whl", hash = "sha256:e212e8b7579031dd2e512168b617373bc1e03888d41ac4e04039240a292d478d"},
{file = "kubernetes-30.1.0.tar.gz", hash = "sha256:41e4c77af9f28e7a6c314e3bd06a8c6229ddd787cad684e0ab9f69b498e98ebc"},
{file = "kubernetes-31.0.0-py2.py3-none-any.whl", hash = "sha256:bf141e2d380c8520eada8b351f4e319ffee9636328c137aa432bc486ca1200e1"},
{file = "kubernetes-31.0.0.tar.gz", hash = "sha256:28945de906c8c259c1ebe62703b56a03b714049372196f854105afe4e6d014c0"},
]
[package.dependencies]
certifi = ">=14.05.14"
durationpy = ">=0.7"
google-auth = ">=1.0.1"
oauthlib = ">=3.2.2"
python-dateutil = ">=2.5.3"
@@ -2536,13 +2565,13 @@ dev = ["bumpver", "isort", "mypy", "pylint", "pytest", "yapf"]
[[package]]
name = "msgraph-sdk"
version = "1.6.0"
version = "1.8.0"
description = "The Microsoft Graph Python SDK"
optional = false
python-versions = ">=3.8"
files = [
{file = "msgraph_sdk-1.6.0-py3-none-any.whl", hash = "sha256:00dc0037a0a15a00bf04a9fd1340bb14b70dd0bbd5bc881bfde4ec9c0e2b9dbc"},
{file = "msgraph_sdk-1.6.0.tar.gz", hash = "sha256:6724b186564b5a208e5069adff4a491a5401f36ed6928edcc4b6702b82b44c94"},
{file = "msgraph_sdk-1.8.0-py3-none-any.whl", hash = "sha256:22a8e4a63f989865228f66a54501bef8105909c7156fe0a079ca9b5296339cc2"},
{file = "msgraph_sdk-1.8.0.tar.gz", hash = "sha256:1ac84bd47ea288a84f46f6c6d0c89d164ee3453b917615632652344538098314"},
]
[package.dependencies]
@@ -2876,40 +2905,53 @@ files = [
[[package]]
name = "pandas"
version = "2.2.2"
version = "2.2.3"
description = "Powerful data structures for data analysis, time series, and statistics"
optional = false
python-versions = ">=3.9"
files = [
{file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"},
{file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"},
{file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"},
{file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"},
{file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"},
{file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"},
{file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"},
{file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"},
{file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"},
{file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"},
{file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"},
{file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"},
{file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"},
{file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"},
{file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"},
{file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"},
{file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"},
{file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"},
{file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"},
{file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"},
{file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"},
{file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"},
{file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"},
{file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"},
{file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"},
{file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"},
{file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"},
{file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"},
{file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"},
{file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"},
{file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"},
{file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"},
{file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"},
{file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"},
{file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"},
{file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"},
{file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"},
{file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"},
{file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"},
{file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"},
{file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"},
{file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"},
{file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"},
{file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"},
{file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"},
{file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"},
{file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"},
{file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"},
{file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"},
{file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"},
{file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"},
{file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"},
{file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"},
{file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"},
{file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"},
{file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"},
{file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"},
{file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"},
{file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"},
{file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"},
{file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"},
{file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"},
{file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"},
{file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"},
{file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"},
{file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"},
{file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"},
{file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"},
{file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"},
{file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"},
{file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"},
]
[package.dependencies]
@@ -3184,16 +3226,17 @@ develop = false
[package.dependencies]
alive-progress = "3.1.5"
awsipranges = "0.3.3"
azure-identity = "1.17.1"
azure-identity = "1.18.0"
azure-keyvault-keys = "4.9.0"
azure-mgmt-applicationinsights = "4.0.0"
azure-mgmt-authorization = "4.0.0"
azure-mgmt-compute = "33.0.0"
azure-mgmt-containerservice = "31.0.0"
azure-mgmt-cosmosdb = "9.5.1"
azure-mgmt-containerregistry = "10.3.0"
azure-mgmt-containerservice = "32.0.0"
azure-mgmt-cosmosdb = "9.6.0"
azure-mgmt-keyvault = "10.3.1"
azure-mgmt-monitor = "6.0.2"
azure-mgmt-network = "26.0.0"
azure-mgmt-network = "27.0.0"
azure-mgmt-rdbms = "10.1.0"
azure-mgmt-resource = "23.1.1"
azure-mgmt-security = "7.0.0"
@@ -3201,29 +3244,29 @@ azure-mgmt-sql = "3.0.1"
azure-mgmt-storage = "21.2.1"
azure-mgmt-subscription = "3.1.1"
azure-mgmt-web = "7.3.1"
azure-storage-blob = "12.22.0"
boto3 = "1.34.162"
botocore = "1.34.162"
azure-storage-blob = "12.23.1"
boto3 = "1.35.28"
botocore = "1.35.29"
colorama = "0.4.6"
cryptography = "43.0.1"
dash = "2.18.0"
dash = "2.18.1"
dash-bootstrap-components = "1.6.0"
detect-secrets = "1.5.0"
google-api-python-client = "2.144.0"
google-api-python-client = "2.147.0"
google-auth-httplib2 = ">=0.1,<0.3"
jsonschema = "4.23.0"
kubernetes = "30.1.0"
kubernetes = "31.0.0"
microsoft-kiota-abstractions = "1.3.3"
msgraph-sdk = "1.6.0"
msgraph-sdk = "1.8.0"
numpy = "2.0.2"
pandas = "2.2.2"
pandas = "2.2.3"
py-ocsf-models = "0.1.1"
pydantic = "1.10.15"
python-dateutil = "^2.9.0.post0"
pytz = "2024.1"
pytz = "2024.2"
schema = "0.7.7"
shodan = "1.31.0"
slack-sdk = "3.32.0"
slack-sdk = "3.33.1"
tabulate = "0.9.0"
tzlocal = "5.2"
@@ -3231,7 +3274,7 @@ tzlocal = "5.2"
type = "git"
url = "https://github.com/prowler-cloud/prowler.git"
reference = "master"
resolved_reference = "1ef15f0b24bdf07b5ce901f3c7c85d1e7c645246"
resolved_reference = "38e024216cd51e2f24b3210ec9008df7f1e54cac"
[[package]]
name = "psutil"
@@ -3747,13 +3790,13 @@ files = [
[[package]]
name = "pytz"
version = "2024.1"
version = "2024.2"
description = "World timezone definitions, modern and historical"
optional = false
python-versions = "*"
files = [
{file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"},
{file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"},
{file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"},
{file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"},
]
[[package]]
@@ -4330,13 +4373,13 @@ files = [
[[package]]
name = "slack-sdk"
version = "3.32.0"
version = "3.33.1"
description = "The Slack API Platform SDK for Python"
optional = false
python-versions = ">=3.6"
files = [
{file = "slack_sdk-3.32.0-py2.py3-none-any.whl", hash = "sha256:f35e85f2847e6c25cf7c2d1df206ca0ad75556263fb592457bf03cca68ef64bb"},
{file = "slack_sdk-3.32.0.tar.gz", hash = "sha256:af8fc4ef1d1cbcecd28d01acf6955a3bb5b13d56f0a43a1b1c7e3b212cc5ec5b"},
{file = "slack_sdk-3.33.1-py2.py3-none-any.whl", hash = "sha256:ef93beec3ce9c8f64da02fd487598a05ec4bc9c92ceed58f122dbe632691cbe2"},
{file = "slack_sdk-3.33.1.tar.gz", hash = "sha256:e328bb661d95db5f66b993b1d64288ac7c72201a745b4c7cf8848dafb7b74e40"},
]
[package.extras]
src/backend/api/db_utils.py
+9 -1
View File
@@ -2,7 +2,7 @@ from contextlib import contextmanager
from django.conf import settings
from django.contrib.auth.models import BaseUserManager
from django.db import models
from django.db import models, transaction, connection
from psycopg2 import connect as psycopg2_connect
from psycopg2.extensions import new_type, register_type, register_adapter, AsIs
@@ -40,6 +40,14 @@ def psycopg_connection(database_alias: str):
psycopg2_connection.close()
@contextmanager
def tenant_transaction(tenant_id: str):
with transaction.atomic():
with connection.cursor() as cursor:
cursor.execute(f"SELECT set_config('api.tenant_id', '{tenant_id}', TRUE);")
yield cursor
class CustomUserManager(BaseUserManager):
def create_user(self, email, password=None, **extra_fields):
if not email:
src/backend/api/middleware.py
+10 -14
View File
@@ -1,22 +1,16 @@
import logging
import time
from django.http import HttpRequest
from config.custom_logging import BackendLogger
def extract_tenant_id(request: HttpRequest) -> str | None:
"""
Extract the tenant ID from the request headers.
Args:
request (HttpRequest): The HTTP request object.
Returns:
str: The tenant ID if present in the headers, otherwise None.
"""
return request.headers.get("X-Tenant-ID")
def extract_auth_info(request) -> dict:
if not hasattr(request, "auth") or request.auth is not None:
tenant_id = request.auth.get("tenant_id", "N/A")
user_id = request.auth.get("user_id", "N/A")
else:
tenant_id, user_id = "N/A", "N/A"
return {"tenant_id": tenant_id, "user_id": user_id}
class APILoggingMiddleware:
@@ -38,15 +32,17 @@ class APILoggingMiddleware:
response = self.get_response(request)
duration = time.time() - request_start_time
auth_info = extract_auth_info(request)
self.logger.info(
"",
extra={
"user_id": auth_info["user_id"],
"tenant_id": auth_info["tenant_id"],
"method": request.method,
"path": request.path,
"query_params": request.GET.dict(),
"status_code": response.status_code,
"duration": duration,
"tenant_id": extract_tenant_id(request),
},
)
src/backend/api/migrations/0001_initial.py
+81 -70
View File
@@ -420,6 +420,70 @@ class Migration(migrations.Migration):
name="unique_provider_uids",
),
),
migrations.CreateModel(
name="Task",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
(
"task_runner_task",
models.OneToOneField(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="task",
related_query_name="task",
to="django_celery_results.taskresult",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "tasks",
"abstract": False,
},
),
migrations.AddConstraint(
model_name="task",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_task",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AddIndex(
model_name="task",
index=models.Index(
fields=["id", "task_runner_task"],
name="tasks_id_trt_id_idx",
),
),
migrations.RunSQL(
f"""
ALTER TABLE {TASK_RUNNER_DB_TABLE} ENABLE ROW LEVEL SECURITY;
CREATE POLICY "{DB_PROWLER_USER}_{TASK_RUNNER_DB_TABLE}_select"
ON {TASK_RUNNER_DB_TABLE}
FOR SELECT
TO {DB_PROWLER_USER}
USING (
task_id::uuid in (SELECT id FROM tasks WHERE tenant_id = (NULLIF(current_setting('{POSTGRES_TENANT_VAR}', true), ''))::uuid)
);
GRANT SELECT ON TABLE {TASK_RUNNER_DB_TABLE} TO {DB_PROWLER_USER};
"""
),
# Create and register ScanTriggerEnum type
migrations.RunPython(
ScanTriggerEnumMigration.create_enum_type,
@@ -488,6 +552,17 @@ class Migration(migrations.Migration):
to="api.provider",
),
),
(
"task",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="scans",
related_query_name="scan",
to="api.task",
null=True,
blank=True,
),
),
(
"tenant",
models.ForeignKey(
@@ -515,70 +590,6 @@ class Migration(migrations.Migration):
name="scans_prov_state_trig_sche_idx",
),
),
migrations.CreateModel(
name="Task",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
(
"task_runner_task",
models.OneToOneField(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="task",
related_query_name="task",
to="django_celery_results.taskresult",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "tasks",
"abstract": False,
},
),
migrations.AddConstraint(
model_name="task",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_task",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AddIndex(
model_name="task",
index=models.Index(
fields=["id", "task_runner_task"],
name="tasks_id_trt_id_idx",
),
),
migrations.RunSQL(
f"""
ALTER TABLE {TASK_RUNNER_DB_TABLE} ENABLE ROW LEVEL SECURITY;
CREATE POLICY "{DB_PROWLER_USER}_{TASK_RUNNER_DB_TABLE}_select"
ON {TASK_RUNNER_DB_TABLE}
FOR SELECT
TO {DB_PROWLER_USER}
USING (
task_id::uuid in (SELECT id FROM tasks WHERE tenant_id = (NULLIF(current_setting('{POSTGRES_TENANT_VAR}', true), ''))::uuid)
);
GRANT SELECT ON TABLE {TASK_RUNNER_DB_TABLE} TO {DB_PROWLER_USER};
"""
),
# Resources
migrations.RunSQL(
sql="""
@@ -812,7 +823,7 @@ class Migration(migrations.Migration):
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_resourcetag",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AddConstraint(
@@ -827,13 +838,13 @@ class Migration(migrations.Migration):
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_resourcetagmapping",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AddConstraint(
model_name="resource",
constraint=models.UniqueConstraint(
fields=("tenant_id", "provider_id", "uid"),
fields=("tenant_id", "provider_id", "uid", "region"),
name="unique_resources_by_provider",
),
),
@@ -842,7 +853,7 @@ class Migration(migrations.Migration):
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_resource",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
# Create and register ScanTypeEnum type
@@ -1084,7 +1095,7 @@ class Migration(migrations.Migration):
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_resourcefindingmapping",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
PostgresAddDefaultPartition(
@@ -1097,7 +1108,7 @@ class Migration(migrations.Migration):
"tenant_id",
name="rls_on_resource_finding_mappings_default",
partition_name="default",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
migrations.AlterModelOptions(
src/backend/api/models.py
+64 -55
View File
@@ -216,56 +216,6 @@ class Provider(RowLevelSecurityProtectedModel):
]
class Scan(RowLevelSecurityProtectedModel):
class TriggerChoices(models.TextChoices):
SCHEDULED = "scheduled", _("Scheduled")
MANUAL = "manual", _("Manual")
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
name = models.CharField(
blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)]
)
provider = models.ForeignKey(
Provider,
on_delete=models.CASCADE,
related_name="scans",
related_query_name="scan",
)
trigger = ScanTriggerEnumField(
choices=TriggerChoices.choices,
)
state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE)
unique_resource_count = models.IntegerField(default=0)
progress = models.IntegerField(default=0)
scanner_args = models.JSONField(default=dict)
duration = models.IntegerField(null=True, blank=True)
scheduled_at = models.DateTimeField(null=True, blank=True)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
# TODO: task foreign key
# TODO: mutelist foreign key
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "scans"
constraints = [
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
indexes = [
models.Index(
fields=["provider", "state", "trigger", "scheduled_at"],
name="scans_prov_state_trig_sche_idx",
),
]
class Task(RowLevelSecurityProtectedModel):
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
@@ -297,6 +247,63 @@ class Task(RowLevelSecurityProtectedModel):
]
class Scan(RowLevelSecurityProtectedModel):
class TriggerChoices(models.TextChoices):
SCHEDULED = "scheduled", _("Scheduled")
MANUAL = "manual", _("Manual")
id = models.UUIDField(primary_key=True, default=uuid7, editable=False)
name = models.CharField(
blank=True, null=True, max_length=100, validators=[MinLengthValidator(3)]
)
provider = models.ForeignKey(
Provider,
on_delete=models.CASCADE,
related_name="scans",
related_query_name="scan",
)
task = models.ForeignKey(
Task,
on_delete=models.CASCADE,
related_name="scans",
related_query_name="scan",
null=True,
blank=True,
)
trigger = ScanTriggerEnumField(
choices=TriggerChoices.choices,
)
state = StateEnumField(choices=StateChoices.choices, default=StateChoices.AVAILABLE)
unique_resource_count = models.IntegerField(default=0)
progress = models.IntegerField(default=0)
scanner_args = models.JSONField(default=dict)
duration = models.IntegerField(null=True, blank=True)
scheduled_at = models.DateTimeField(null=True, blank=True)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
started_at = models.DateTimeField(null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
# TODO: mutelist foreign key
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "scans"
constraints = [
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
indexes = [
models.Index(
fields=["provider", "state", "trigger", "scheduled_at"],
name="scans_prov_state_trig_sche_idx",
),
]
class ResourceTag(RowLevelSecurityProtectedModel):
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
@@ -329,7 +336,7 @@ class ResourceTag(RowLevelSecurityProtectedModel):
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
@@ -409,13 +416,13 @@ class Resource(RowLevelSecurityProtectedModel):
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "provider_id", "uid"),
fields=("tenant_id", "provider_id", "uid", "region"),
name="unique_resources_by_provider",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
@@ -444,7 +451,7 @@ class ResourceTagMapping(RowLevelSecurityProtectedModel):
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
@@ -485,9 +492,11 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel):
impact_extended = models.TextField(blank=True, null=True)
raw_result = models.JSONField(default=dict)
# TODO: review usability
tags = models.JSONField(default=dict, null=True, blank=True)
check_id = models.CharField(max_length=100, blank=False, null=False)
# TODO: review usability
check_metadata = models.JSONField(default=dict, null=False)
# Relationships
@@ -591,6 +600,6 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT"],
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
src/backend/api/specs/v1.yaml
+56 -80
View File
@@ -651,7 +651,7 @@ paths:
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/SerializerMetaclassResponse'
$ref: '#/components/schemas/OpenApiResponseResponse'
description: ''
/api/v1/providers/{id}/connection:
post:
@@ -676,7 +676,7 @@ paths:
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/SerializerMetaclassResponse'
$ref: '#/components/schemas/OpenApiResponseResponse'
description: ''
/api/v1/resources:
get:
@@ -1013,6 +1013,7 @@ paths:
- scanner_args
- duration
- provider
- task
- started_at
- completed_at
- scheduled_at
@@ -1181,11 +1182,11 @@ paths:
security:
- jwtAuth: []
responses:
'201':
'202':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/ScanCreateResponse'
$ref: '#/components/schemas/OpenApiResponseResponse'
description: ''
/api/v1/scans/{id}:
get:
@@ -1208,6 +1209,7 @@ paths:
- scanner_args
- duration
- provider
- task
- started_at
- completed_at
- scheduled_at
@@ -1267,7 +1269,7 @@ paths:
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/SerializerMetaclassResponse'
$ref: '#/components/schemas/ScanUpdateResponse'
description: ''
/api/v1/tasks:
get:
@@ -1415,7 +1417,7 @@ paths:
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/SerializerMetaclassResponse'
$ref: '#/components/schemas/OpenApiResponseResponse'
description: ''
/api/v1/tenants:
get:
@@ -1781,7 +1783,8 @@ paths:
/api/v1/tokens/refresh:
post:
operationId: tokens_refresh_create
description: Refresh an access token by providing a valid refresh token.
description: Refresh an access token by providing a valid refresh token. Former
refresh tokens are invalidated when a new one is issued.
summary: Refresh a token
tags:
- Token
@@ -2111,35 +2114,6 @@ paths:
description: ''
components:
schemas:
DelayedTask:
type: object
required:
- type
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/TypeB52Enum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
attributes:
type: object
properties:
id:
type: string
state:
type: string
enum:
- available
- scheduled
- executing
- completed
- failed
- cancelled
readOnly: true
required:
- id
Finding:
type: object
required:
@@ -2376,6 +2350,13 @@ components:
type: string
enum:
- Membership
OpenApiResponseResponse:
type: object
properties:
data:
$ref: '#/components/schemas/Task'
required:
- data
PaginatedFindingList:
type: object
properties:
@@ -2988,33 +2969,7 @@ components:
- data
description: The identifier of the related object.
title: Resource Identifier
required:
- provider
ScanCreate:
type: object
required:
- type
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/Type2bbEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
attributes:
type: object
properties:
scanner_args: {}
name:
type: string
nullable: true
maxLength: 100
minLength: 3
relationships:
type: object
properties:
provider:
task:
type: object
properties:
data:
@@ -3026,7 +2981,7 @@ components:
type:
type: string
enum:
- Provider
- Task
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
@@ -3038,6 +2993,7 @@ components:
- data
description: The identifier of the related object.
title: Resource Identifier
nullable: true
required:
- provider
ScanCreateRequest:
@@ -3096,13 +3052,6 @@ components:
- provider
required:
- data
ScanCreateResponse:
type: object
properties:
data:
$ref: '#/components/schemas/ScanCreate'
required:
- data
ScanResponse:
type: object
properties:
@@ -3110,6 +3059,37 @@ components:
$ref: '#/components/schemas/Scan'
required:
- data
ScanUpdate:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/Type2bbEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id:
type: string
format: uuid
attributes:
type: object
properties:
name:
type: string
nullable: true
maxLength: 100
minLength: 3
ScanUpdateResponse:
type: object
properties:
data:
$ref: '#/components/schemas/ScanUpdate'
required:
- data
SerializerMetaclassResponse:
type: object
properties:
@@ -3121,18 +3101,14 @@ components:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
allOf:
- $ref: '#/components/schemas/TypeB52Enum'
- $ref: '#/components/schemas/TaskTypeEnum'
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
id:
type: string
format: uuid
attributes:
type: object
properties:
@@ -3170,6 +3146,10 @@ components:
$ref: '#/components/schemas/Task'
required:
- data
TaskTypeEnum:
type: string
enum:
- Task
Tenant:
type: object
required:
@@ -3466,10 +3446,6 @@ components:
type: string
enum:
- User
TypeB52Enum:
type: string
enum:
- Task
User:
type: object
required:
src/backend/api/tests/test_middleware.py
+45 -24
View File
@@ -1,33 +1,54 @@
from unittest.mock import patch, Mock, ANY
from unittest.mock import MagicMock, patch
import pytest
from django.http import HttpResponse
from django.test import RequestFactory
from api.middleware import APILoggingMiddleware
@pytest.mark.django_db
@patch("logging.getLogger")
def test_api_logger_middleware(mock_get_logger, client):
mock_logger = Mock()
mock_get_logger.side_effect = lambda name: mock_logger if name == "api" else Mock()
request_method = "GET"
request_path = "/this_path_does_not_exist"
tenant_id = "12646005-9067-4d2a-a098-8bb378604362"
api_logger = "api"
def test_api_logging_middleware_logging(mock_logger):
factory = RequestFactory()
response = getattr(client, request_method.lower())(
request_path, headers={"X-Tenant-ID": tenant_id}
)
request = factory.get("/test-path?param1=value1&param2=value2")
request.method = "GET"
mock_get_logger.assert_any_call(api_logger)
mock_logger.info.assert_called_once_with(
"",
extra={
"method": request_method,
"path": request_path,
"query_params": {},
"status_code": response.status_code,
"duration": ANY,
"tenant_id": tenant_id,
},
)
response = HttpResponse()
response.status_code = 200
assert isinstance(mock_logger.info.call_args[1]["extra"]["duration"], float)
get_response = MagicMock(return_value=response)
with patch("api.middleware.extract_auth_info") as mock_extract_auth_info:
mock_extract_auth_info.return_value = {
"user_id": "user123",
"tenant_id": "tenant456",
}
with patch("api.middleware.logging.getLogger") as mock_get_logger:
mock_logger = MagicMock()
mock_get_logger.return_value = mock_logger
middleware = APILoggingMiddleware(get_response)
with patch("api.middleware.time.time") as mock_time:
mock_time.side_effect = [1000.0, 1001.0] # Start time and end time
middleware(request)
get_response.assert_called_once_with(request)
mock_extract_auth_info.assert_called_once_with(request)
expected_extra = {
"user_id": "user123",
"tenant_id": "tenant456",
"method": "GET",
"path": "/test-path",
"query_params": {"param1": "value1", "param2": "value2"},
"status_code": 200,
"duration": 1.0,
}
mock_logger.info.assert_called_once_with("", extra=expected_extra)
src/backend/api/tests/test_views.py
+24 -3
View File
@@ -926,13 +926,21 @@ class TestProviderViewSet:
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
@patch("api.v1.views.Task.objects.get")
@patch("api.v1.views.delete_provider_task.delay")
def test_providers_delete(
self, mock_delete_task, authenticated_client, providers_fixture
self,
mock_delete_task,
mock_task_get,
authenticated_client,
providers_fixture,
tasks_fixture,
):
prowler_task = tasks_fixture[0]
task_mock = Mock()
task_mock.id = "12345"
task_mock.id = prowler_task.id
mock_delete_task.return_value = task_mock
mock_task_get.return_value = prowler_task
provider1, *_ = providers_fixture
response = authenticated_client.delete(
@@ -951,17 +959,22 @@ class TestProviderViewSet:
)
assert response.status_code == status.HTTP_404_NOT_FOUND
@patch("api.v1.views.Task.objects.get")
@patch("api.v1.views.check_provider_connection_task.delay")
def test_providers_connection(
self,
mock_provider_connection,
mock_task_get,
authenticated_client,
providers_fixture,
tasks_fixture,
):
prowler_task = tasks_fixture[0]
task_mock = Mock()
task_mock.id = "12345"
task_mock.id = prowler_task.id
task_mock.status = "PENDING"
mock_provider_connection.return_value = task_mock
mock_task_get.return_value = prowler_task
provider1, *_ = providers_fixture
assert provider1.connected is None
@@ -1127,13 +1140,21 @@ class TestScanViewSet:
),
],
)
@patch("api.v1.views.Task.objects.get")
@patch("api.v1.views.perform_scan_task.delay")
def test_scans_create_valid(
self,
mock_perform_scan_task,
mock_task_get,
authenticated_client,
scan_json_payload,
expected_scanner_args,
providers_fixture,
tasks_fixture,
):
prowler_task = tasks_fixture[0]
mock_perform_scan_task.return_value.id = prowler_task.id
mock_task_get.return_value = prowler_task
*_, provider5 = providers_fixture
# Provider5 has these scanner_args
# scanner_args={"key1": "value1", "key2": {"key21": "value21"}}
src/backend/api/v1/serializers.py
+31 -27
View File
@@ -86,7 +86,7 @@ class TokenSerializer(serializers.Serializer):
except Exception as e:
raise serializers.ValidationError({"detail": str(e)})
return {"refresh": str(refresh), "access": str(access)}
return {"access": str(access), "refresh": str(refresh)}
class TokenRefreshSerializer(serializers.Serializer):
@@ -215,7 +215,7 @@ class UserUpdateSerializer(BaseWriteSerializer):
# Tasks
class TaskBase(serializers.Serializer):
class TaskBase(serializers.ModelSerializer):
state_mapping = {
"PENDING": StateChoices.SCHEDULED,
"STARTED": StateChoices.EXECUTING,
@@ -225,8 +225,9 @@ class TaskBase(serializers.Serializer):
"REVOKED": StateChoices.CANCELLED,
}
class JSONAPIMeta:
resource_name = "Task"
class Meta:
fields = ["id"]
model = Task
def map_state(self, task_result_state):
return self.state_mapping.get(task_result_state, StateChoices.AVAILABLE)
@@ -244,27 +245,6 @@ class TaskBase(serializers.Serializer):
return self.map_state(task_result_state)
class DelayedTaskSerializer(TaskBase):
id = serializers.CharField()
state = serializers.SerializerMethodField(read_only=True)
class Meta:
fields = [
"id",
"state",
]
@extend_schema_field(
{
"type": "string",
"enum": StateChoices.values,
}
)
def get_state(self, obj):
task_result_state = obj.status if obj else None
return self.map_state(task_result_state)
class TaskSerializer(RLSSerializer, TaskBase):
state = serializers.SerializerMethodField(read_only=True)
metadata = serializers.SerializerMethodField(read_only=True)
@@ -444,6 +424,7 @@ class ScanSerializer(RLSSerializer):
"scanner_args",
"duration",
"provider",
"task",
"started_at",
"completed_at",
"scheduled_at",
@@ -455,7 +436,7 @@ class ScanCreateSerializer(RLSSerializer, BaseWriteSerializer):
class Meta:
model = Scan
# TODO: add mutelist when implemented
fields = ["provider", "scanner_args", "name"]
fields = ["id", "provider", "scanner_args", "name"]
def create(self, validated_data):
provider = validated_data.get("provider")
@@ -488,9 +469,32 @@ class ScanUpdateSerializer(BaseWriteSerializer):
}
class ScanTaskSerializer(RLSSerializer):
trigger = serializers.ChoiceField(
choices=Scan.TriggerChoices.choices, read_only=True
)
state = StateEnumSerializerField(read_only=True)
class Meta:
model = Scan
fields = [
"id",
"name",
"trigger",
"state",
"unique_resource_count",
"progress",
"scanner_args",
"duration",
"started_at",
"completed_at",
"scheduled_at",
]
class ResourceTagSerializer(RLSSerializer):
"""
Serializer fore the ResourceTag model
Serializer for the ResourceTag model
"""
class Meta:
src/backend/api/v1/views.py
+64 -32
View File
@@ -7,7 +7,12 @@ from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_control
from drf_spectacular.settings import spectacular_settings
from drf_spectacular.utils import OpenApiTypes
from drf_spectacular.utils import extend_schema, extend_schema_view, OpenApiParameter
from drf_spectacular.utils import (
extend_schema,
extend_schema_view,
OpenApiParameter,
OpenApiResponse,
)
from drf_spectacular.views import SpectacularAPIView
from rest_framework import status, permissions
from rest_framework.decorators import action
@@ -42,14 +47,17 @@ from api.v1.serializers import (
ProviderUpdateSerializer,
TenantSerializer,
TaskSerializer,
DelayedTaskSerializer,
ScanSerializer,
ScanCreateSerializer,
ScanUpdateSerializer,
ResourceSerializer,
FindingSerializer,
)
from tasks.tasks import check_provider_connection_task, delete_provider_task
from tasks.tasks import (
check_provider_connection_task,
delete_provider_task,
perform_scan_task,
)
CACHE_DECORATOR = cache_control(
max_age=django_settings.CACHE_MAX_AGE,
@@ -390,7 +398,7 @@ class TenantMembersViewSet(BaseTenantViewset):
tags=["Provider"],
summary="Delete a provider",
description="Remove a provider from the system by their ID.",
responses={202: DelayedTaskSerializer},
responses={202: OpenApiResponse(response=TaskSerializer)},
),
)
@method_decorator(CACHE_DECORATOR, name="list")
@@ -420,7 +428,7 @@ class ProviderViewSet(BaseRLSViewSet):
elif self.action == "partial_update":
return ProviderUpdateSerializer
elif self.action in ["connection", "destroy"]:
return DelayedTaskSerializer
return TaskSerializer
return super().get_serializer_class()
def partial_update(self, request, *args, **kwargs):
@@ -443,7 +451,7 @@ class ProviderViewSet(BaseRLSViewSet):
summary="Check connection",
description="Try to verify connection. For instance, Role & Credentials are set correctly",
request=None,
responses={202: DelayedTaskSerializer},
responses={202: OpenApiResponse(response=TaskSerializer)},
)
@action(detail=True, methods=["post"], url_name="connection")
def connection(self, request, pk=None):
@@ -451,24 +459,30 @@ class ProviderViewSet(BaseRLSViewSet):
task = check_provider_connection_task.delay(
provider_id=pk, tenant_id=request.tenant_id
)
serializer = DelayedTaskSerializer(task)
prowler_task = Task.objects.get(id=task.id)
serializer = TaskSerializer(prowler_task)
return Response(
data=serializer.data,
status=status.HTTP_202_ACCEPTED,
headers={
"Content-Location": reverse("task-detail", kwargs={"pk": task.id})
"Content-Location": reverse(
"task-detail", kwargs={"pk": prowler_task.id}
)
},
)
def destroy(self, request, *args, pk=None, **kwargs):
get_object_or_404(Provider, pk=pk)
task = delete_provider_task.delay(provider_id=pk, tenant_id=request.tenant_id)
serializer = DelayedTaskSerializer(task)
prowler_task = Task.objects.get(id=task.id)
serializer = TaskSerializer(prowler_task)
return Response(
data=serializer.data,
status=status.HTTP_202_ACCEPTED,
headers={
"Content-Location": reverse("task-detail", kwargs={"pk": task.id})
"Content-Location": reverse(
"task-detail", kwargs={"pk": prowler_task.id}
)
},
)
@@ -482,6 +496,10 @@ class ProviderViewSet(BaseRLSViewSet):
summary="Retrieve data from a specific scan",
description="Fetch detailed information about a specific scan by its ID.",
),
partial_update=extend_schema(
summary="Partially update a scan",
description="Update certain fields of an existing scan without affecting other fields.",
),
create=extend_schema(
summary="Trigger a manual scan",
description=(
@@ -491,12 +509,8 @@ class ProviderViewSet(BaseRLSViewSet):
"merged with the provider's defaults. This means that your provided settings will override "
"the defaults only where they conflict, while the rest of the default settings will remain intact."
),
),
partial_update=extend_schema(
summary="Partially update a scan",
description="Update certain fields of an existing scan without affecting other fields.",
request=ScanUpdateSerializer,
responses={200: ScanSerializer},
request=ScanCreateSerializer,
responses={202: OpenApiResponse(response=TaskSerializer)},
),
)
@method_decorator(CACHE_DECORATOR, name="list")
@@ -521,25 +535,13 @@ class ScanViewSet(BaseRLSViewSet):
def get_serializer_class(self):
if self.action == "create":
if hasattr(self, "response_serializer_class"):
return self.response_serializer_class
return ScanCreateSerializer
elif self.action == "partial_update":
return ScanUpdateSerializer
return super().get_serializer_class()
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
scan = serializer.save()
# TODO: Run scan through task and return task info here
return Response(
data=serializer.data,
status=status.HTTP_202_ACCEPTED,
headers={
"Content-Location": reverse("scan-detail", kwargs={"pk": scan.id})
},
)
def partial_update(self, request, *args, **kwargs):
instance = self.get_object()
serializer = self.get_serializer(
@@ -555,6 +557,35 @@ class ScanViewSet(BaseRLSViewSet):
)
return Response(data=read_serializer.data, status=status.HTTP_200_OK)
def create(self, request, *args, **kwargs):
input_serializer = self.get_serializer(data=request.data)
input_serializer.is_valid(raise_exception=True)
scan = input_serializer.save()
task = perform_scan_task.delay(
tenant_id=request.tenant_id,
scan_id=str(scan.id),
provider_id=str(scan.provider_id),
checks_to_execute=scan.scanner_args.get("checks_to_execute", []),
)
scan.task_id = task.id
scan.save(update_fields=["task_id"])
prowler_task = Task.objects.get(id=task.id)
self.response_serializer_class = TaskSerializer
output_serializer = self.get_serializer(prowler_task)
return Response(
data=output_serializer.data,
status=status.HTTP_202_ACCEPTED,
headers={
"Content-Location": reverse(
"task-detail", kwargs={"pk": prowler_task.id}
)
},
)
@extend_schema_view(
list=extend_schema(
@@ -569,7 +600,7 @@ class ScanViewSet(BaseRLSViewSet):
tags=["Task"],
summary="Revoke a task",
description="Try to revoke a task using its ID. Only tasks that are not yet in progress can be revoked.",
responses={202: DelayedTaskSerializer},
responses={202: OpenApiResponse(response=TaskSerializer)},
),
)
class TaskViewSet(BaseRLSViewSet):
@@ -602,7 +633,8 @@ class TaskViewSet(BaseRLSViewSet):
task_instance = AsyncResult(pk)
task_instance.revoke()
serializer = DelayedTaskSerializer(task_instance)
task.refresh_from_db()
serializer = TaskSerializer(task)
return Response(
data=serializer.data,
status=status.HTTP_202_ACCEPTED,
src/backend/config/custom_logging.py
+8 -4
View File
@@ -45,6 +45,10 @@ class NDJSONFormatter(logging.Formatter):
}
# Add REST API extra fields
if hasattr(record, "user_id"):
log_record["user_id"] = record.user_id
if hasattr(record, "tenant_id"):
log_record["tenant_id"] = record.tenant_id
if hasattr(record, "method"):
log_record["method"] = record.method
if hasattr(record, "path"):
@@ -55,8 +59,6 @@ class NDJSONFormatter(logging.Formatter):
log_record["duration"] = record.duration
if hasattr(record, "status_code"):
log_record["status_code"] = record.status_code
if hasattr(record, "tenant_id"):
log_record["tenant_id"] = record.tenant_id
if record.exc_info:
log_record["exc_info"] = self.formatException(record.exc_info)
@@ -82,13 +84,15 @@ class HumanReadableFormatter(logging.Formatter):
f"function={record.funcName}",
f"process={record.process}",
f"thread={record.thread}",
f"transaction-id={record.transaction_id if hasattr(record, "transaction_id") else None}]",
f"transaction-id={record.transaction_id if hasattr(record, 'transaction_id') else None}]",
f"{record.getMessage()}",
]
# Add REST API extra fields
if hasattr(record, "user_id"):
log_components.append(f"({record.user_id})")
if hasattr(record, "tenant_id"):
log_components.append(f"({record.tenant_id})")
log_components.append(f"[{record.tenant_id}]")
if hasattr(record, "method"):
log_components.append(f'"{record.method} {record.path}"')
if hasattr(record, "query_params"):
src/backend/tasks/jobs/connection.py
+18 -9
View File
@@ -2,6 +2,9 @@ from datetime import datetime, timezone
from celery.utils.log import get_task_logger
from prowler.providers.aws.aws_provider import AwsProvider
from prowler.providers.azure.azure_provider import AzureProvider
from prowler.providers.gcp.gcp_provider import GcpProvider
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
from api.models import Provider
@@ -26,20 +29,26 @@ def check_provider_connection(provider_id: str):
"""
provider_instance = Provider.objects.get(pk=provider_id)
match provider_instance.provider:
# TODO Refactor when proper credentials are implemented
case Provider.ProviderChoices.AWS.value:
# TODO Refactor when proper credentials are implemented
try:
connection_result = AwsProvider.test_connection(
raise_on_exception=False
)
# TODO: Improve this exception handling when SDK exceptions are implemented
except Exception as e:
logger.warning(str(e))
raise e
prowler_provider = AwsProvider
case Provider.ProviderChoices.GCP.value:
prowler_provider = GcpProvider
case Provider.ProviderChoices.AZURE.value:
prowler_provider = AzureProvider
case Provider.ProviderChoices.KUBERNETES.value:
prowler_provider = KubernetesProvider
case _:
raise ValueError(
f"Provider type {provider_instance.provider} not supported"
)
try:
connection_result = prowler_provider.test_connection(raise_on_exception=False)
except Exception as e:
logger.warning(
f"Unexpected exception checking {provider_instance.provider} provider connection: {str(e)}"
)
raise e
provider_instance.connected = connection_result.is_connected
provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc)
src/backend/tasks/jobs/scan.py
+146
View File
@@ -0,0 +1,146 @@
import json
import time
from datetime import datetime, timezone
from celery.utils.log import get_task_logger
from prowler.lib.outputs.finding import Finding as ProwlerFinding
from prowler.lib.scan.scan import Scan as ProwlerScan
from prowler.providers.aws.aws_provider import AwsProvider
from prowler.providers.azure.azure_provider import AzureProvider
from prowler.providers.gcp.gcp_provider import GcpProvider
from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider
from api.db_utils import tenant_transaction
from api.models import (
Provider,
Scan,
Finding,
Resource,
ResourceTag,
StatusChoices as FindingStatus,
StateChoices,
)
from api.v1.serializers import ScanTaskSerializer
from config.celery import TaskTimeoutError
logger = get_task_logger(__name__)
def store_finding(
finding: ProwlerFinding,
tenant_id: str,
scan_instance: Scan,
resource_instance: Resource,
) -> Finding:
with tenant_transaction(tenant_id):
finding_instance = Finding.objects.create(
tenant_id=tenant_id,
delta=Finding.DeltaChoices.NEW,
status=FindingStatus[finding.status.value],
status_extended=finding.status_extended,
severity=finding.severity.value,
impact=finding.severity.value,
raw_result=json.loads(finding.json()),
check_id=finding.check_id,
scan=scan_instance,
)
finding_instance.add_resources([resource_instance])
return finding_instance
def store_resources(
finding: ProwlerFinding, tenant_id: str, provider_instance: Provider
) -> tuple[Resource, tuple[str, str]]:
with tenant_transaction(tenant_id):
resource_instance, _ = Resource.objects.get_or_create(
tenant_id=tenant_id,
provider=provider_instance,
uid=finding.resource_uid,
region=finding.region,
service=finding.service_name,
type=finding.resource_type,
)
with tenant_transaction(tenant_id):
tags = [
ResourceTag.objects.get_or_create(
tenant_id=tenant_id, key=key, value=value
)[0]
for key, value in finding.resource_tags.items()
]
resource_instance.upsert_or_delete_tags(tags=tags)
return resource_instance, (resource_instance.uid, resource_instance.region)
def perform_prowler_scan(
tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None
):
with tenant_transaction(tenant_id):
exception = None
provider_instance = Provider.objects.get(pk=provider_id)
start_time = time.time()
unique_resources = set()
# Prevent race conditions
while not Scan.objects.filter(id=scan_id).exists():
if time.time() - start_time > 10:
raise TaskTimeoutError(
f"Could not find scan with given id {scan_id} within 10 seconds"
)
time.sleep(0.1)
scan_instance = Scan.objects.get(pk=scan_id)
scan_instance.state = StateChoices.EXECUTING
scan_instance.started_at = datetime.now(tz=timezone.utc)
scan_instance.save()
try:
match provider_instance.provider:
case Provider.ProviderChoices.AWS.value:
prowler_provider = AwsProvider
case Provider.ProviderChoices.GCP.value:
prowler_provider = GcpProvider
case Provider.ProviderChoices.AZURE.value:
prowler_provider = AzureProvider
case Provider.ProviderChoices.KUBERNETES.value:
prowler_provider = KubernetesProvider
case _:
raise ValueError(
f"Provider type {provider_instance.provider} not supported"
)
with tenant_transaction(tenant_id):
connection_status = prowler_provider.test_connection(
raise_on_exception=False
)
provider_instance.connected = connection_status.is_connected
provider_instance.connection_last_checked_at = datetime.now(tz=timezone.utc)
provider_instance.save()
if connection_status.is_connected is False:
raise ValueError(f"Provider {provider_instance.provider} is not connected")
prowler_scan = ProwlerScan(
provider=prowler_provider(), checks_to_execute=checks_to_execute or []
)
for progress, findings in prowler_scan.scan():
for finding in findings:
resource_instance, resource_uid_tuple = store_resources(
finding, tenant_id, provider_instance
)
store_finding(finding, tenant_id, scan_instance, resource_instance)
unique_resources.add(resource_uid_tuple)
with tenant_transaction(tenant_id):
scan_instance.progress = progress
scan_instance.save()
scan_instance.state = StateChoices.COMPLETED
except Exception as e:
logger.error(f"Error performing scan {scan_id}: {e}")
exception = e
scan_instance.state = StateChoices.FAILED
finally:
with tenant_transaction(tenant_id):
scan_instance.duration = time.time() - start_time
scan_instance.completed_at = datetime.now(tz=timezone.utc)
scan_instance.unique_resource_count = len(unique_resources)
scan_instance.save()
if exception is not None:
raise exception
serializer = ScanTaskSerializer(instance=scan_instance)
return serializer.data
src/backend/tasks/tasks.py
+29
View File
@@ -5,6 +5,7 @@ from api.models import Provider
from config.celery import RLSTask
from tasks.jobs.connection import check_provider_connection
from tasks.jobs.deletion import delete_instance
from tasks.jobs.scan import perform_prowler_scan
@shared_task(base=RLSTask, name="provider-connection-check")
@@ -40,3 +41,31 @@ def delete_provider_task(provider_id: str):
including related models if cascading deletes were triggered.
"""
return delete_instance(model=Provider, pk=provider_id)
@shared_task(base=RLSTask, name="scan-perform")
def perform_scan_task(
tenant_id: str, scan_id: str, provider_id: str, checks_to_execute: list[str] = None
):
"""
Task to perform a Prowler scan on a given provider.
This task runs a Prowler scan on the provider identified by `provider_id`
under the tenant identified by `tenant_id`. The scan will use the `scan_id`
for tracking purposes.
Args:
tenant_id (str): The tenant ID under which the scan is being performed.
scan_id (str): The ID of the scan to be performed.
provider_id (str): The primary key of the Provider instance to scan.
checks_to_execute (list[str], optional): A list of specific checks to perform during the scan. Defaults to None.
Returns:
dict: The result of the scan execution, typically including the status and results of the performed checks.
"""
return perform_prowler_scan(
tenant_id=tenant_id,
scan_id=scan_id,
provider_id=provider_id,
checks_to_execute=checks_to_execute,
)
src/backend/tasks/tests/test_scan.py
+111
View File
@@ -0,0 +1,111 @@
import json
from unittest.mock import patch, MagicMock, PropertyMock
import pytest
from api.models import StateChoices, StatusChoices, Severity, Finding, Resource
from tasks.jobs.scan import perform_prowler_scan
@pytest.mark.django_db
class TestPerformScan:
@patch("tasks.jobs.scan.ProwlerScan")
@patch("tasks.jobs.scan.AwsProvider")
@patch("api.db_utils.tenant_transaction")
def test_perform_prowler_scan_success(
self,
mock_tenant_transaction,
mock_aws_provider,
mock_prowler_scan,
tenants_fixture,
scans_fixture,
providers_fixture,
):
assert len(Finding.objects.all()) == 0
assert len(Resource.objects.all()) == 0
tenant, _ = tenants_fixture
scan, *_ = scans_fixture
provider, *_ = providers_fixture
tenant_id = str(tenant.id)
scan_id = str(scan.id)
provider_id = str(provider.id)
checks_to_execute = ["check1", "check2"]
connection_status_mock = MagicMock()
type(connection_status_mock).is_connected = PropertyMock(return_value=True)
mock_aws_provider.test_connection.return_value = connection_status_mock
mock_aws_provider_instance = mock_aws_provider.return_value
mock_aws_provider_instance.test_connection.return_value = connection_status_mock
finding = MagicMock()
finding.status = StatusChoices.PASS
finding.status_extended = "test status extended"
finding.severity = Severity.medium
finding.check_id = "check1"
finding.json.return_value = '{"key": "value"}'
finding.resource_uid = "resource_uid"
finding.region = "region"
finding.service_name = "service_name"
finding.resource_type = "resource_type"
mock_prowler_scan.return_value.scan.return_value = [(100, [finding])]
perform_prowler_scan(tenant_id, scan_id, provider_id, checks_to_execute)
scan.refresh_from_db()
scan_finding = Finding.objects.get(scan=scan)
scan_resource = Resource.objects.get(provider=provider)
assert scan.tenant == tenant
assert scan.provider == provider
assert scan.state == StateChoices.COMPLETED
assert scan.completed_at is not None
assert scan.duration is not None
assert scan.started_at is not None
assert scan.unique_resource_count == 1
assert scan.progress == 100
assert scan_finding.status == finding.status
assert scan_finding.status_extended == finding.status_extended
assert scan_finding.severity == finding.severity
assert scan_finding.check_id == finding.check_id
assert scan_finding.raw_result == json.loads(finding.json())
assert scan_resource.tenant == tenant
assert scan_resource.uid == finding.resource_uid
assert scan_resource.region == finding.region
assert scan_resource.service == finding.service_name
assert scan_resource.type == finding.resource_type
@patch("tasks.jobs.scan.ProwlerScan")
@patch("tasks.jobs.scan.AwsProvider")
@patch("api.db_utils.tenant_transaction")
def test_perform_prowler_scan_no_connection(
self,
mock_tenant_transaction,
mock_aws_provider,
mock_prowler_scan,
tenants_fixture,
scans_fixture,
providers_fixture,
):
tenant, _ = tenants_fixture
scan, *_ = scans_fixture
provider, *_ = providers_fixture
tenant_id = str(tenant.id)
scan_id = str(scan.id)
provider_id = str(provider.id)
checks_to_execute = ["check1", "check2"]
connection_status_mock = MagicMock()
type(connection_status_mock).is_connected = PropertyMock(return_value=False)
mock_aws_provider.test_connection.return_value = connection_status_mock
mock_aws_provider_instance = mock_aws_provider.return_value
mock_aws_provider_instance.test_connection.return_value = connection_status_mock
with pytest.raises(ValueError):
perform_prowler_scan(tenant_id, scan_id, provider_id, checks_to_execute)
scan.refresh_from_db()
assert scan.state == StateChoices.FAILED