mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-07-23 20:42:02 +00:00
feat(findings): Improve /findings/metadata performance (#6749)
Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>
This commit is contained in:
Prowler Bot
Generated
+95
-13
@@ -252,6 +252,20 @@ files = [
|
||||
[package.dependencies]
|
||||
cryptography = "*"
|
||||
|
||||
[[package]]
|
||||
name = "autopep8"
|
||||
version = "2.3.2"
|
||||
description = "A tool that automatically formats Python code to conform to the PEP 8 style guide"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128"},
|
||||
{file = "autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
pycodestyle = ">=2.12.0"
|
||||
|
||||
[[package]]
|
||||
name = "awsipranges"
|
||||
version = "0.3.3"
|
||||
@@ -1488,6 +1502,23 @@ docs = ["Sphinx (==2.2.0)", "docutils (<0.18)", "sphinx-rtd-theme (==0.4.3)"]
|
||||
publish = ["build (==0.7.0)", "twine (==3.7.1)"]
|
||||
test = ["coveralls (==3.3.0)", "dj-database-url (==0.5.0)", "freezegun (==1.1.0)", "psycopg2 (>=2.8.4,<3.0.0)", "pytest (==6.2.5)", "pytest-benchmark (==3.4.1)", "pytest-cov (==3.0.0)", "pytest-django (==4.4.0)", "pytest-freezegun (==0.4.2)", "pytest-lazy-fixture (==0.6.3)", "snapshottest (==0.6.0)", "tox (==3.24.4)"]
|
||||
|
||||
[[package]]
|
||||
name = "django-silk"
|
||||
version = "5.3.2"
|
||||
description = "Silky smooth profiling for the Django Framework"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "django_silk-5.3.2-py3-none-any.whl", hash = "sha256:49f1caebfda28b1707f0cfef524e0476beb82b8c5e40f5ccff7f73a6b4f6d3ac"},
|
||||
{file = "django_silk-5.3.2.tar.gz", hash = "sha256:b0db54eebedb8d16f572321bd6daccac0bd3f547ae2618bb45d96fe8fc02229d"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
autopep8 = "*"
|
||||
Django = ">=4.2"
|
||||
gprof2dot = ">=2017.09.19"
|
||||
sqlparse = "*"
|
||||
|
||||
[[package]]
|
||||
name = "django-timezone-field"
|
||||
version = "7.1"
|
||||
@@ -1984,6 +2015,17 @@ protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4
|
||||
[package.extras]
|
||||
grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"]
|
||||
|
||||
[[package]]
|
||||
name = "gprof2dot"
|
||||
version = "2024.6.6"
|
||||
description = "Generate a dot graph from the output of several profilers."
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"},
|
||||
{file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "grapheme"
|
||||
version = "0.6.0"
|
||||
@@ -2627,13 +2669,13 @@ portalocker = ">=1.4,<3"
|
||||
|
||||
[[package]]
|
||||
name = "msgraph-core"
|
||||
version = "1.2.0"
|
||||
version = "1.2.1"
|
||||
description = "Core component of the Microsoft Graph Python SDK"
|
||||
optional = false
|
||||
python-versions = ">=3.9"
|
||||
files = [
|
||||
{file = "msgraph_core-1.2.0-py3-none-any.whl", hash = "sha256:4ce14bbe743c0f2dd8b53c7fcd338fc14081e0df6b14021f0d0e4bb63cd5a840"},
|
||||
{file = "msgraph_core-1.2.0.tar.gz", hash = "sha256:a4e42f692e664c60d63359e610bbf990f57b42d8080417261ff7042bbd59c98b"},
|
||||
{file = "msgraph_core-1.2.1-py3-none-any.whl", hash = "sha256:4591c1dc4359a323a50b2d29e4cb75ecaff32b4b6781f1ae743a655fb4a20ad7"},
|
||||
{file = "msgraph_core-1.2.1.tar.gz", hash = "sha256:87a3cb4d36dad590a3f02aaedf422547cbac10460bd9f0b6c984fab9556150d3"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -2849,6 +2891,32 @@ files = [
|
||||
{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "narwhals"
|
||||
version = "1.24.1"
|
||||
description = "Extremely lightweight compatibility layer between dataframe libraries"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "narwhals-1.24.1-py3-none-any.whl", hash = "sha256:d8983fe14851c95d60576ddca37c094bd4ed24ab9ea98396844fb20ad9aaf184"},
|
||||
{file = "narwhals-1.24.1.tar.gz", hash = "sha256:b09b8253d945f23cdb683a84685abf3afb9f96114d89e9f35dc876e143f65007"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
core = ["duckdb", "pandas", "polars", "pyarrow", "pyarrow-stubs"]
|
||||
cudf = ["cudf (>=24.10.0)"]
|
||||
dask = ["dask[dataframe] (>=2024.8)"]
|
||||
dev = ["covdefaults", "hypothesis", "pre-commit", "pytest", "pytest-cov", "pytest-env", "pytest-randomly", "typing-extensions"]
|
||||
docs = ["black", "duckdb", "jinja2", "markdown-exec[ansi]", "mkdocs", "mkdocs-autorefs", "mkdocs-material", "mkdocstrings[python]", "pandas", "polars (>=1.0.0)", "pyarrow"]
|
||||
duckdb = ["duckdb (>=1.0)"]
|
||||
extra = ["scikit-learn"]
|
||||
ibis = ["ibis-framework (>=6.0.0)", "packaging", "pyarrow-hotfix", "rich"]
|
||||
modin = ["modin"]
|
||||
pandas = ["pandas (>=0.25.3)"]
|
||||
polars = ["polars (>=0.20.3)"]
|
||||
pyarrow = ["pyarrow (>=11.0.0)"]
|
||||
pyspark = ["pyspark (>=3.5.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "nest-asyncio"
|
||||
version = "1.6.0"
|
||||
@@ -3200,18 +3268,21 @@ type = ["mypy (>=1.11.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "plotly"
|
||||
version = "5.24.1"
|
||||
version = "6.0.0"
|
||||
description = "An open-source, interactive data visualization library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "plotly-5.24.1-py3-none-any.whl", hash = "sha256:f67073a1e637eb0dc3e46324d9d51e2fe76e9727c892dde64ddf1e1b51f29089"},
|
||||
{file = "plotly-5.24.1.tar.gz", hash = "sha256:dbc8ac8339d248a4bcc36e08a5659bacfe1b079390b8953533f4eb22169b4bae"},
|
||||
{file = "plotly-6.0.0-py3-none-any.whl", hash = "sha256:f708871c3a9349a68791ff943a5781b1ec04de7769ea69068adcd9202e57653a"},
|
||||
{file = "plotly-6.0.0.tar.gz", hash = "sha256:c4aad38b8c3d65e4a5e7dd308b084143b9025c2cc9d5317fc1f1d30958db87d3"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
narwhals = ">=1.15.1"
|
||||
packaging = "*"
|
||||
tenacity = ">=6.2.0"
|
||||
|
||||
[package.extras]
|
||||
express = ["numpy"]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
@@ -3391,7 +3462,7 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "prowler"
|
||||
version = "5.2.1"
|
||||
version = "5.2.2"
|
||||
description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks."
|
||||
optional = false
|
||||
python-versions = ">=3.9,<3.13"
|
||||
@@ -3450,7 +3521,7 @@ tzlocal = "5.2"
|
||||
type = "git"
|
||||
url = "https://github.com/prowler-cloud/prowler.git"
|
||||
reference = "v5.2"
|
||||
resolved_reference = "1fe125867c9879619a6e0ed50d69b1a9c7f7935e"
|
||||
resolved_reference = "1a5428445aaea28b4a6425f7aa8c76838f46f171"
|
||||
|
||||
[[package]]
|
||||
name = "psutil"
|
||||
@@ -3603,6 +3674,17 @@ files = [
|
||||
[package.dependencies]
|
||||
pyasn1 = ">=0.4.6,<0.7.0"
|
||||
|
||||
[[package]]
|
||||
name = "pycodestyle"
|
||||
version = "2.12.1"
|
||||
description = "Python style guide checker"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3"},
|
||||
{file = "pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycparser"
|
||||
version = "2.22"
|
||||
@@ -4943,13 +5025,13 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "xlsxwriter"
|
||||
version = "3.2.1"
|
||||
version = "3.2.2"
|
||||
description = "A Python module for creating Excel XLSX files."
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "XlsxWriter-3.2.1-py3-none-any.whl", hash = "sha256:7e8f7c60b7a1660ef791d46ab5de78469cb978b991ca841af61f5832d2f9f4fe"},
|
||||
{file = "XlsxWriter-3.2.1.tar.gz", hash = "sha256:97618759cb264fb6a93397f660cca156ffa9561743b1823dafb60dc4474e1902"},
|
||||
{file = "XlsxWriter-3.2.2-py3-none-any.whl", hash = "sha256:272ce861e7fa5e82a4a6ebc24511f2cb952fde3461f6c6e1a1e81d3272db1471"},
|
||||
{file = "xlsxwriter-3.2.2.tar.gz", hash = "sha256:befc7f92578a85fed261639fb6cde1fd51b79c5e854040847dde59d4317077dc"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5070,4 +5152,4 @@ type = ["pytest-mypy"]
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.11,<3.13"
|
||||
content-hash = "6470908880bf38f32a95d7c7cc624af343f6615945cb9cd92158672aa5371bc7"
|
||||
content-hash = "4d015c14bb849873c82fd3804c55013c51860689bba99fa095b9896062903357"
|
||||
|
||||
+2
-1
@@ -8,7 +8,7 @@ description = "Prowler's API (Django/DRF)"
|
||||
license = "Apache-2.0"
|
||||
name = "prowler-api"
|
||||
package-mode = false
|
||||
version = "1.3.1"
|
||||
version = "1.3.2"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
celery = {extras = ["pytest"], version = "^5.4.0"}
|
||||
@@ -37,6 +37,7 @@ uuid6 = "2024.7.10"
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
bandit = "1.7.9"
|
||||
coverage = "7.5.4"
|
||||
django-silk = "5.3.2"
|
||||
docker = "7.1.0"
|
||||
freezegun = "1.5.1"
|
||||
mypy = "1.10.1"
|
||||
|
||||
@@ -4,13 +4,17 @@ class MainRouter:
|
||||
|
||||
def db_for_read(self, model, **hints): # noqa: F841
|
||||
model_table_name = model._meta.db_table
|
||||
if model_table_name.startswith("django_"):
|
||||
if model_table_name.startswith("django_") or model_table_name.startswith(
|
||||
"silk_"
|
||||
):
|
||||
return self.admin_db
|
||||
return None
|
||||
|
||||
def db_for_write(self, model, **hints): # noqa: F841
|
||||
model_table_name = model._meta.db_table
|
||||
if model_table_name.startswith("django_"):
|
||||
if model_table_name.startswith("django_") or model_table_name.startswith(
|
||||
"silk_"
|
||||
):
|
||||
return self.admin_db
|
||||
return None
|
||||
|
||||
|
||||
@@ -319,26 +319,27 @@ class FindingFilter(FilterSet):
|
||||
field_name="resources__type", lookup_expr="icontains"
|
||||
)
|
||||
|
||||
resource_tag_key = CharFilter(field_name="resources__tags__key")
|
||||
resource_tag_key__in = CharInFilter(
|
||||
field_name="resources__tags__key", lookup_expr="in"
|
||||
)
|
||||
resource_tag_key__icontains = CharFilter(
|
||||
field_name="resources__tags__key", lookup_expr="icontains"
|
||||
)
|
||||
resource_tag_value = CharFilter(field_name="resources__tags__value")
|
||||
resource_tag_value__in = CharInFilter(
|
||||
field_name="resources__tags__value", lookup_expr="in"
|
||||
)
|
||||
resource_tag_value__icontains = CharFilter(
|
||||
field_name="resources__tags__value", lookup_expr="icontains"
|
||||
)
|
||||
resource_tags = CharInFilter(
|
||||
method="filter_resource_tag",
|
||||
lookup_expr="in",
|
||||
help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be "
|
||||
"separated by commas.",
|
||||
)
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# resource_tag_key = CharFilter(field_name="resources__tags__key")
|
||||
# resource_tag_key__in = CharInFilter(
|
||||
# field_name="resources__tags__key", lookup_expr="in"
|
||||
# )
|
||||
# resource_tag_key__icontains = CharFilter(
|
||||
# field_name="resources__tags__key", lookup_expr="icontains"
|
||||
# )
|
||||
# resource_tag_value = CharFilter(field_name="resources__tags__value")
|
||||
# resource_tag_value__in = CharInFilter(
|
||||
# field_name="resources__tags__value", lookup_expr="in"
|
||||
# )
|
||||
# resource_tag_value__icontains = CharFilter(
|
||||
# field_name="resources__tags__value", lookup_expr="icontains"
|
||||
# )
|
||||
# resource_tags = CharInFilter(
|
||||
# method="filter_resource_tag",
|
||||
# lookup_expr="in",
|
||||
# help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be "
|
||||
# "separated by commas.",
|
||||
# )
|
||||
|
||||
scan = UUIDFilter(method="filter_scan_id")
|
||||
scan__in = UUIDInFilter(method="filter_scan_id_in")
|
||||
@@ -374,12 +375,6 @@ class FindingFilter(FilterSet):
|
||||
},
|
||||
}
|
||||
|
||||
@property
|
||||
def qs(self):
|
||||
# Force distinct results to prevent duplicates with many-to-many relationships
|
||||
parent_qs = super().qs
|
||||
return parent_qs.distinct()
|
||||
|
||||
# Convert filter values to UUIDv7 values for use with partitioning
|
||||
def filter_scan_id(self, queryset, name, value):
|
||||
try:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: Prowler API
|
||||
version: 1.3.1
|
||||
version: 1.3.2
|
||||
description: |-
|
||||
Prowler API specification.
|
||||
|
||||
@@ -478,51 +478,6 @@ paths:
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tag_key]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_key__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_key__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tag_value]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_value__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_value__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tags]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: |-
|
||||
Filter by resource tags `key:value` pairs.
|
||||
Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_type]
|
||||
schema:
|
||||
@@ -1028,51 +983,6 @@ paths:
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tag_key]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_key__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_key__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tag_value]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_value__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_value__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tags]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: |-
|
||||
Filter by resource tags `key:value` pairs.
|
||||
Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_type]
|
||||
schema:
|
||||
@@ -1280,7 +1190,6 @@ paths:
|
||||
- services
|
||||
- regions
|
||||
- resource_types
|
||||
- tags
|
||||
description: endpoint return only specific fields in the response on a per-type
|
||||
basis by including a fields[TYPE] query parameter.
|
||||
explode: false
|
||||
@@ -1498,51 +1407,6 @@ paths:
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tag_key]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_key__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_key__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tag_value]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_value__icontains]
|
||||
schema:
|
||||
type: string
|
||||
- in: query
|
||||
name: filter[resource_tag_value__in]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_tags]
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
description: |-
|
||||
Filter by resource tags `key:value` pairs.
|
||||
Multiple values may be separated by commas.
|
||||
explode: false
|
||||
style: form
|
||||
- in: query
|
||||
name: filter[resource_type]
|
||||
schema:
|
||||
@@ -6081,13 +5945,10 @@ components:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
tags:
|
||||
description: Tags are described as key-value pairs.
|
||||
required:
|
||||
- services
|
||||
- regions
|
||||
- resource_types
|
||||
- tags
|
||||
FindingMetadataResponse:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -2454,15 +2454,16 @@ class TestFindingViewSet:
|
||||
("search", "ec2", 2),
|
||||
# full text search on finding tags
|
||||
("search", "value2", 2),
|
||||
("resource_tag_key", "key", 2),
|
||||
("resource_tag_key__in", "key,key2", 2),
|
||||
("resource_tag_key__icontains", "key", 2),
|
||||
("resource_tag_value", "value", 2),
|
||||
("resource_tag_value__in", "value,value2", 2),
|
||||
("resource_tag_value__icontains", "value", 2),
|
||||
("resource_tags", "key:value", 2),
|
||||
("resource_tags", "not:exists", 0),
|
||||
("resource_tags", "not:exists,key:value", 2),
|
||||
# Temporary disabled until we implement tag filtering in the UI
|
||||
# ("resource_tag_key", "key", 2),
|
||||
# ("resource_tag_key__in", "key,key2", 2),
|
||||
# ("resource_tag_key__icontains", "key", 2),
|
||||
# ("resource_tag_value", "value", 2),
|
||||
# ("resource_tag_value__in", "value,value2", 2),
|
||||
# ("resource_tag_value__icontains", "value", 2),
|
||||
# ("resource_tags", "key:value", 2),
|
||||
# ("resource_tags", "not:exists", 0),
|
||||
# ("resource_tags", "not:exists,key:value", 2),
|
||||
]
|
||||
),
|
||||
)
|
||||
@@ -2611,7 +2612,8 @@ class TestFindingViewSet:
|
||||
|
||||
expected_services = {"ec2", "s3"}
|
||||
expected_regions = {"eu-west-1", "us-east-1"}
|
||||
expected_tags = {"key": ["value"], "key2": ["value2"]}
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# expected_tags = {"key": ["value"], "key2": ["value2"]}
|
||||
expected_resource_types = {"prowler-test"}
|
||||
|
||||
assert data["data"]["type"] == "findings-metadata"
|
||||
@@ -2621,7 +2623,7 @@ class TestFindingViewSet:
|
||||
assert (
|
||||
set(data["data"]["attributes"]["resource_types"]) == expected_resource_types
|
||||
)
|
||||
assert data["data"]["attributes"]["tags"] == expected_tags
|
||||
# assert data["data"]["attributes"]["tags"] == expected_tags
|
||||
|
||||
def test_findings_metadata_severity_retrieve(
|
||||
self, authenticated_client, findings_fixture
|
||||
@@ -2638,7 +2640,8 @@ class TestFindingViewSet:
|
||||
|
||||
expected_services = {"s3"}
|
||||
expected_regions = {"eu-west-1"}
|
||||
expected_tags = {"key": ["value"], "key2": ["value2"]}
|
||||
# Temporary disabled until we implement tag filtering in the UI
|
||||
# expected_tags = {"key": ["value"], "key2": ["value2"]}
|
||||
expected_resource_types = {"prowler-test"}
|
||||
|
||||
assert data["data"]["type"] == "findings-metadata"
|
||||
@@ -2648,7 +2651,7 @@ class TestFindingViewSet:
|
||||
assert (
|
||||
set(data["data"]["attributes"]["resource_types"]) == expected_resource_types
|
||||
)
|
||||
assert data["data"]["attributes"]["tags"] == expected_tags
|
||||
# assert data["data"]["attributes"]["tags"] == expected_tags
|
||||
|
||||
def test_findings_metadata_future_date(self, authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
@@ -2660,7 +2663,8 @@ class TestFindingViewSet:
|
||||
assert data["data"]["id"] is None
|
||||
assert data["data"]["attributes"]["services"] == []
|
||||
assert data["data"]["attributes"]["regions"] == []
|
||||
assert data["data"]["attributes"]["tags"] == {}
|
||||
# Temporary disabled until we implement tag filtering in the UI
|
||||
# assert data["data"]["attributes"]["tags"] == {}
|
||||
assert data["data"]["attributes"]["resource_types"] == []
|
||||
|
||||
def test_findings_metadata_invalid_date(self, authenticated_client):
|
||||
|
||||
@@ -933,7 +933,8 @@ class FindingMetadataSerializer(serializers.Serializer):
|
||||
resource_types = serializers.ListField(
|
||||
child=serializers.CharField(), allow_empty=True
|
||||
)
|
||||
tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tags = serializers.JSONField(help_text="Tags are described as key-value pairs.")
|
||||
|
||||
class Meta:
|
||||
resource_name = "findings-metadata"
|
||||
|
||||
@@ -1,30 +1,31 @@
|
||||
from django.conf import settings
|
||||
from django.urls import include, path
|
||||
from drf_spectacular.views import SpectacularRedocView
|
||||
from rest_framework_nested import routers
|
||||
|
||||
from api.v1.views import (
|
||||
ComplianceOverviewViewSet,
|
||||
CustomTokenObtainView,
|
||||
CustomTokenRefreshView,
|
||||
FindingViewSet,
|
||||
MembershipViewSet,
|
||||
ProviderGroupViewSet,
|
||||
ProviderGroupProvidersRelationshipView,
|
||||
ProviderSecretViewSet,
|
||||
InvitationViewSet,
|
||||
InvitationAcceptViewSet,
|
||||
RoleViewSet,
|
||||
RoleProviderGroupRelationshipView,
|
||||
UserRoleRelationshipView,
|
||||
InvitationViewSet,
|
||||
MembershipViewSet,
|
||||
OverviewViewSet,
|
||||
ComplianceOverviewViewSet,
|
||||
ProviderGroupProvidersRelationshipView,
|
||||
ProviderGroupViewSet,
|
||||
ProviderSecretViewSet,
|
||||
ProviderViewSet,
|
||||
ResourceViewSet,
|
||||
RoleProviderGroupRelationshipView,
|
||||
RoleViewSet,
|
||||
ScanViewSet,
|
||||
ScheduleViewSet,
|
||||
SchemaView,
|
||||
TaskViewSet,
|
||||
TenantMembersViewSet,
|
||||
TenantViewSet,
|
||||
UserRoleRelationshipView,
|
||||
UserViewSet,
|
||||
)
|
||||
|
||||
@@ -112,3 +113,6 @@ urlpatterns = [
|
||||
path("schema", SchemaView.as_view(), name="schema"),
|
||||
path("docs", SpectacularRedocView.as_view(url_name="schema"), name="docs"),
|
||||
]
|
||||
|
||||
if settings.DEBUG:
|
||||
urlpatterns += [path("silk/", include("silk.urls", namespace="silk"))]
|
||||
|
||||
@@ -4,7 +4,7 @@ from django.contrib.postgres.aggregates import ArrayAgg
|
||||
from django.contrib.postgres.search import SearchQuery
|
||||
from django.db import transaction
|
||||
from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum
|
||||
from django.db.models.functions import Coalesce, JSONObject
|
||||
from django.db.models.functions import Coalesce
|
||||
from django.urls import reverse
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.views.decorators.cache import cache_control
|
||||
@@ -193,7 +193,7 @@ class SchemaView(SpectacularAPIView):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
spectacular_settings.TITLE = "Prowler API"
|
||||
spectacular_settings.VERSION = "1.3.1"
|
||||
spectacular_settings.VERSION = "1.3.2"
|
||||
spectacular_settings.DESCRIPTION = (
|
||||
"Prowler API specification.\n\nThis file is auto-generated."
|
||||
)
|
||||
@@ -1392,48 +1392,59 @@ class FindingViewSet(BaseRLSViewSet):
|
||||
|
||||
@action(detail=False, methods=["get"], url_name="metadata")
|
||||
def metadata(self, request):
|
||||
tenant_id = self.request.tenant_id
|
||||
queryset = self.get_queryset()
|
||||
filtered_queryset = self.filter_queryset(queryset)
|
||||
|
||||
result = filtered_queryset.aggregate(
|
||||
services=ArrayAgg("resources__service", flat=True, distinct=True),
|
||||
regions=ArrayAgg("resources__region", flat=True, distinct=True),
|
||||
tags=ArrayAgg(
|
||||
JSONObject(
|
||||
key=F("resources__tags__key"), value=F("resources__tags__value")
|
||||
),
|
||||
distinct=True,
|
||||
filter=Q(resources__tags__key__isnull=False),
|
||||
),
|
||||
resource_types=ArrayAgg("resources__type", flat=True, distinct=True),
|
||||
)
|
||||
if result["services"] is None:
|
||||
result["services"] = []
|
||||
if result["regions"] is None:
|
||||
result["regions"] = []
|
||||
if result["regions"] is None:
|
||||
result["regions"] = []
|
||||
if result["resource_types"] is None:
|
||||
result["resource_types"] = []
|
||||
if result["tags"] is None:
|
||||
result["tags"] = []
|
||||
relevant_resources = Resource.objects.filter(
|
||||
tenant_id=tenant_id, findings__in=filtered_queryset
|
||||
).distinct()
|
||||
|
||||
tags_dict = {}
|
||||
for t in result["tags"]:
|
||||
key, value = t["key"], t["value"]
|
||||
if key not in tags_dict:
|
||||
tags_dict[key] = []
|
||||
tags_dict[key].append(value)
|
||||
|
||||
result["tags"] = tags_dict
|
||||
|
||||
serializer = self.get_serializer(
|
||||
data=result,
|
||||
services = (
|
||||
relevant_resources.values_list("service", flat=True)
|
||||
.distinct()
|
||||
.order_by("service")
|
||||
)
|
||||
|
||||
regions = (
|
||||
relevant_resources.exclude(region="")
|
||||
.values_list("region", flat=True)
|
||||
.distinct()
|
||||
.order_by("region")
|
||||
)
|
||||
|
||||
resource_types = (
|
||||
relevant_resources.values_list("type", flat=True)
|
||||
.distinct()
|
||||
.order_by("type")
|
||||
)
|
||||
|
||||
# Temporarily disabled until we implement tag filtering in the UI
|
||||
# tag_data = (
|
||||
# relevant_resources
|
||||
# .filter(tags__key__isnull=False, tags__value__isnull=False)
|
||||
# .exclude(tags__key="")
|
||||
# .exclude(tags__value="")
|
||||
# .values("tags__key", "tags__value")
|
||||
# .distinct()
|
||||
# .order_by("tags__key", "tags__value")
|
||||
# )
|
||||
#
|
||||
# tags_dict = {}
|
||||
# for row in tag_data:
|
||||
# k, v = row["tags__key"], row["tags__value"]
|
||||
# tags_dict.setdefault(k, []).append(v)
|
||||
|
||||
result = {
|
||||
"services": list(services),
|
||||
"regions": list(regions),
|
||||
"resource_types": list(resource_types),
|
||||
# "tags": tags_dict
|
||||
}
|
||||
|
||||
serializer = self.get_serializer(data=result)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
return Response(data=serializer.data, status=status.HTTP_200_OK)
|
||||
return Response(serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
|
||||
@@ -37,3 +37,9 @@ REST_FRAMEWORK["DEFAULT_FILTER_BACKENDS"] = tuple( # noqa: F405
|
||||
) + ("api.filters.CustomDjangoFilterBackend",)
|
||||
|
||||
SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0="
|
||||
|
||||
MIDDLEWARE += [ # noqa: F405
|
||||
"silk.middleware.SilkyMiddleware",
|
||||
]
|
||||
|
||||
INSTALLED_APPS += ["silk"] # noqa: F405
|
||||
|
||||
@@ -152,6 +152,9 @@ def perform_prowler_scan(
|
||||
|
||||
for progress, findings in prowler_scan.scan():
|
||||
for finding in findings:
|
||||
if finding is None:
|
||||
logger.error(f"None finding detected on scan {scan_id}.")
|
||||
continue
|
||||
for attempt in range(CELERY_DEADLOCK_ATTEMPTS):
|
||||
try:
|
||||
with rls_transaction(tenant_id):
|
||||
@@ -176,7 +179,10 @@ def perform_prowler_scan(
|
||||
|
||||
# Update resource fields if necessary
|
||||
updated_fields = []
|
||||
if resource_instance.region != finding.region:
|
||||
if (
|
||||
finding.region
|
||||
and resource_instance.region != finding.region
|
||||
):
|
||||
resource_instance.region = finding.region
|
||||
updated_fields.append("region")
|
||||
if resource_instance.service != finding.service_name:
|
||||
|
||||
Reference in New Issue
Block a user