From 84ecd7ab2c6fd69d95d07d0f0246c7fdfb079010 Mon Sep 17 00:00:00 2001 From: Prowler Bot Date: Thu, 30 Jan 2025 13:54:25 +0100 Subject: [PATCH] feat(findings): Improve /findings/metadata performance (#6749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Víctor Fernández Poyatos --- api/poetry.lock | 108 +++++++++++++++--- api/pyproject.toml | 3 +- api/src/backend/api/db_router.py | 8 +- api/src/backend/api/filters.py | 47 ++++---- api/src/backend/api/specs/v1.yaml | 141 +----------------------- api/src/backend/api/tests/test_views.py | 32 +++--- api/src/backend/api/v1/serializers.py | 3 +- api/src/backend/api/v1/urls.py | 22 ++-- api/src/backend/api/v1/views.py | 85 +++++++------- api/src/backend/config/django/devel.py | 6 + api/src/backend/tasks/jobs/scan.py | 8 +- 11 files changed, 219 insertions(+), 244 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 60be88e720..2dca54f321 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -252,6 +252,20 @@ files = [ [package.dependencies] cryptography = "*" +[[package]] +name = "autopep8" +version = "2.3.2" +description = "A tool that automatically formats Python code to conform to the PEP 8 style guide" +optional = false +python-versions = ">=3.9" +files = [ + {file = "autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128"}, + {file = "autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758"}, +] + +[package.dependencies] +pycodestyle = ">=2.12.0" + [[package]] name = "awsipranges" version = "0.3.3" @@ -1488,6 +1502,23 @@ docs = ["Sphinx (==2.2.0)", "docutils (<0.18)", "sphinx-rtd-theme (==0.4.3)"] publish = ["build (==0.7.0)", "twine (==3.7.1)"] test = ["coveralls (==3.3.0)", "dj-database-url (==0.5.0)", "freezegun (==1.1.0)", "psycopg2 (>=2.8.4,<3.0.0)", "pytest (==6.2.5)", "pytest-benchmark (==3.4.1)", "pytest-cov (==3.0.0)", "pytest-django (==4.4.0)", "pytest-freezegun (==0.4.2)", "pytest-lazy-fixture (==0.6.3)", "snapshottest (==0.6.0)", "tox (==3.24.4)"] +[[package]] +name = "django-silk" +version = "5.3.2" +description = "Silky smooth profiling for the Django Framework" +optional = false +python-versions = ">=3.9" +files = [ + {file = "django_silk-5.3.2-py3-none-any.whl", hash = "sha256:49f1caebfda28b1707f0cfef524e0476beb82b8c5e40f5ccff7f73a6b4f6d3ac"}, + {file = "django_silk-5.3.2.tar.gz", hash = "sha256:b0db54eebedb8d16f572321bd6daccac0bd3f547ae2618bb45d96fe8fc02229d"}, +] + +[package.dependencies] +autopep8 = "*" +Django = ">=4.2" +gprof2dot = ">=2017.09.19" +sqlparse = "*" + [[package]] name = "django-timezone-field" version = "7.1" @@ -1984,6 +2015,17 @@ protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4 [package.extras] grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] +[[package]] +name = "gprof2dot" +version = "2024.6.6" +description = "Generate a dot graph from the output of several profilers." +optional = false +python-versions = ">=3.8" +files = [ + {file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"}, + {file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"}, +] + [[package]] name = "grapheme" version = "0.6.0" @@ -2627,13 +2669,13 @@ portalocker = ">=1.4,<3" [[package]] name = "msgraph-core" -version = "1.2.0" +version = "1.2.1" description = "Core component of the Microsoft Graph Python SDK" optional = false python-versions = ">=3.9" files = [ - {file = "msgraph_core-1.2.0-py3-none-any.whl", hash = "sha256:4ce14bbe743c0f2dd8b53c7fcd338fc14081e0df6b14021f0d0e4bb63cd5a840"}, - {file = "msgraph_core-1.2.0.tar.gz", hash = "sha256:a4e42f692e664c60d63359e610bbf990f57b42d8080417261ff7042bbd59c98b"}, + {file = "msgraph_core-1.2.1-py3-none-any.whl", hash = "sha256:4591c1dc4359a323a50b2d29e4cb75ecaff32b4b6781f1ae743a655fb4a20ad7"}, + {file = "msgraph_core-1.2.1.tar.gz", hash = "sha256:87a3cb4d36dad590a3f02aaedf422547cbac10460bd9f0b6c984fab9556150d3"}, ] [package.dependencies] @@ -2849,6 +2891,32 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "narwhals" +version = "1.24.1" +description = "Extremely lightweight compatibility layer between dataframe libraries" +optional = false +python-versions = ">=3.8" +files = [ + {file = "narwhals-1.24.1-py3-none-any.whl", hash = "sha256:d8983fe14851c95d60576ddca37c094bd4ed24ab9ea98396844fb20ad9aaf184"}, + {file = "narwhals-1.24.1.tar.gz", hash = "sha256:b09b8253d945f23cdb683a84685abf3afb9f96114d89e9f35dc876e143f65007"}, +] + +[package.extras] +core = ["duckdb", "pandas", "polars", "pyarrow", "pyarrow-stubs"] +cudf = ["cudf (>=24.10.0)"] +dask = ["dask[dataframe] (>=2024.8)"] +dev = ["covdefaults", "hypothesis", "pre-commit", "pytest", "pytest-cov", "pytest-env", "pytest-randomly", "typing-extensions"] +docs = ["black", "duckdb", "jinja2", "markdown-exec[ansi]", "mkdocs", "mkdocs-autorefs", "mkdocs-material", "mkdocstrings[python]", "pandas", "polars (>=1.0.0)", "pyarrow"] +duckdb = ["duckdb (>=1.0)"] +extra = ["scikit-learn"] +ibis = ["ibis-framework (>=6.0.0)", "packaging", "pyarrow-hotfix", "rich"] +modin = ["modin"] +pandas = ["pandas (>=0.25.3)"] +polars = ["polars (>=0.20.3)"] +pyarrow = ["pyarrow (>=11.0.0)"] +pyspark = ["pyspark (>=3.5.0)"] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -3200,18 +3268,21 @@ type = ["mypy (>=1.11.2)"] [[package]] name = "plotly" -version = "5.24.1" +version = "6.0.0" description = "An open-source, interactive data visualization library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "plotly-5.24.1-py3-none-any.whl", hash = "sha256:f67073a1e637eb0dc3e46324d9d51e2fe76e9727c892dde64ddf1e1b51f29089"}, - {file = "plotly-5.24.1.tar.gz", hash = "sha256:dbc8ac8339d248a4bcc36e08a5659bacfe1b079390b8953533f4eb22169b4bae"}, + {file = "plotly-6.0.0-py3-none-any.whl", hash = "sha256:f708871c3a9349a68791ff943a5781b1ec04de7769ea69068adcd9202e57653a"}, + {file = "plotly-6.0.0.tar.gz", hash = "sha256:c4aad38b8c3d65e4a5e7dd308b084143b9025c2cc9d5317fc1f1d30958db87d3"}, ] [package.dependencies] +narwhals = ">=1.15.1" packaging = "*" -tenacity = ">=6.2.0" + +[package.extras] +express = ["numpy"] [[package]] name = "pluggy" @@ -3391,7 +3462,7 @@ files = [ [[package]] name = "prowler" -version = "5.2.1" +version = "5.2.2" description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks." optional = false python-versions = ">=3.9,<3.13" @@ -3450,7 +3521,7 @@ tzlocal = "5.2" type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "v5.2" -resolved_reference = "1fe125867c9879619a6e0ed50d69b1a9c7f7935e" +resolved_reference = "1a5428445aaea28b4a6425f7aa8c76838f46f171" [[package]] name = "psutil" @@ -3603,6 +3674,17 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6,<0.7.0" +[[package]] +name = "pycodestyle" +version = "2.12.1" +description = "Python style guide checker" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3"}, + {file = "pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521"}, +] + [[package]] name = "pycparser" version = "2.22" @@ -4943,13 +5025,13 @@ files = [ [[package]] name = "xlsxwriter" -version = "3.2.1" +version = "3.2.2" description = "A Python module for creating Excel XLSX files." optional = false python-versions = ">=3.6" files = [ - {file = "XlsxWriter-3.2.1-py3-none-any.whl", hash = "sha256:7e8f7c60b7a1660ef791d46ab5de78469cb978b991ca841af61f5832d2f9f4fe"}, - {file = "XlsxWriter-3.2.1.tar.gz", hash = "sha256:97618759cb264fb6a93397f660cca156ffa9561743b1823dafb60dc4474e1902"}, + {file = "XlsxWriter-3.2.2-py3-none-any.whl", hash = "sha256:272ce861e7fa5e82a4a6ebc24511f2cb952fde3461f6c6e1a1e81d3272db1471"}, + {file = "xlsxwriter-3.2.2.tar.gz", hash = "sha256:befc7f92578a85fed261639fb6cde1fd51b79c5e854040847dde59d4317077dc"}, ] [[package]] @@ -5070,4 +5152,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "6470908880bf38f32a95d7c7cc624af343f6615945cb9cd92158672aa5371bc7" +content-hash = "4d015c14bb849873c82fd3804c55013c51860689bba99fa095b9896062903357" diff --git a/api/pyproject.toml b/api/pyproject.toml index 8333613d1b..8e426e6468 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -8,7 +8,7 @@ description = "Prowler's API (Django/DRF)" license = "Apache-2.0" name = "prowler-api" package-mode = false -version = "1.3.1" +version = "1.3.2" [tool.poetry.dependencies] celery = {extras = ["pytest"], version = "^5.4.0"} @@ -37,6 +37,7 @@ uuid6 = "2024.7.10" [tool.poetry.group.dev.dependencies] bandit = "1.7.9" coverage = "7.5.4" +django-silk = "5.3.2" docker = "7.1.0" freezegun = "1.5.1" mypy = "1.10.1" diff --git a/api/src/backend/api/db_router.py b/api/src/backend/api/db_router.py index f3a986ac20..939672f88e 100644 --- a/api/src/backend/api/db_router.py +++ b/api/src/backend/api/db_router.py @@ -4,13 +4,17 @@ class MainRouter: def db_for_read(self, model, **hints): # noqa: F841 model_table_name = model._meta.db_table - if model_table_name.startswith("django_"): + if model_table_name.startswith("django_") or model_table_name.startswith( + "silk_" + ): return self.admin_db return None def db_for_write(self, model, **hints): # noqa: F841 model_table_name = model._meta.db_table - if model_table_name.startswith("django_"): + if model_table_name.startswith("django_") or model_table_name.startswith( + "silk_" + ): return self.admin_db return None diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 1042c3aa86..0ace11fe9c 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -319,26 +319,27 @@ class FindingFilter(FilterSet): field_name="resources__type", lookup_expr="icontains" ) - resource_tag_key = CharFilter(field_name="resources__tags__key") - resource_tag_key__in = CharInFilter( - field_name="resources__tags__key", lookup_expr="in" - ) - resource_tag_key__icontains = CharFilter( - field_name="resources__tags__key", lookup_expr="icontains" - ) - resource_tag_value = CharFilter(field_name="resources__tags__value") - resource_tag_value__in = CharInFilter( - field_name="resources__tags__value", lookup_expr="in" - ) - resource_tag_value__icontains = CharFilter( - field_name="resources__tags__value", lookup_expr="icontains" - ) - resource_tags = CharInFilter( - method="filter_resource_tag", - lookup_expr="in", - help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be " - "separated by commas.", - ) + # Temporarily disabled until we implement tag filtering in the UI + # resource_tag_key = CharFilter(field_name="resources__tags__key") + # resource_tag_key__in = CharInFilter( + # field_name="resources__tags__key", lookup_expr="in" + # ) + # resource_tag_key__icontains = CharFilter( + # field_name="resources__tags__key", lookup_expr="icontains" + # ) + # resource_tag_value = CharFilter(field_name="resources__tags__value") + # resource_tag_value__in = CharInFilter( + # field_name="resources__tags__value", lookup_expr="in" + # ) + # resource_tag_value__icontains = CharFilter( + # field_name="resources__tags__value", lookup_expr="icontains" + # ) + # resource_tags = CharInFilter( + # method="filter_resource_tag", + # lookup_expr="in", + # help_text="Filter by resource tags `key:value` pairs.\nMultiple values may be " + # "separated by commas.", + # ) scan = UUIDFilter(method="filter_scan_id") scan__in = UUIDInFilter(method="filter_scan_id_in") @@ -374,12 +375,6 @@ class FindingFilter(FilterSet): }, } - @property - def qs(self): - # Force distinct results to prevent duplicates with many-to-many relationships - parent_qs = super().qs - return parent_qs.distinct() - # Convert filter values to UUIDv7 values for use with partitioning def filter_scan_id(self, queryset, name, value): try: diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index cb355b1aa7..7448843b54 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.3.1 + version: 1.3.2 description: |- Prowler API specification. @@ -478,51 +478,6 @@ paths: description: Multiple values may be separated by commas. explode: false style: form - - in: query - name: filter[resource_tag_key] - schema: - type: string - - in: query - name: filter[resource_tag_key__icontains] - schema: - type: string - - in: query - name: filter[resource_tag_key__in] - schema: - type: array - items: - type: string - description: Multiple values may be separated by commas. - explode: false - style: form - - in: query - name: filter[resource_tag_value] - schema: - type: string - - in: query - name: filter[resource_tag_value__icontains] - schema: - type: string - - in: query - name: filter[resource_tag_value__in] - schema: - type: array - items: - type: string - description: Multiple values may be separated by commas. - explode: false - style: form - - in: query - name: filter[resource_tags] - schema: - type: array - items: - type: string - description: |- - Filter by resource tags `key:value` pairs. - Multiple values may be separated by commas. - explode: false - style: form - in: query name: filter[resource_type] schema: @@ -1028,51 +983,6 @@ paths: description: Multiple values may be separated by commas. explode: false style: form - - in: query - name: filter[resource_tag_key] - schema: - type: string - - in: query - name: filter[resource_tag_key__icontains] - schema: - type: string - - in: query - name: filter[resource_tag_key__in] - schema: - type: array - items: - type: string - description: Multiple values may be separated by commas. - explode: false - style: form - - in: query - name: filter[resource_tag_value] - schema: - type: string - - in: query - name: filter[resource_tag_value__icontains] - schema: - type: string - - in: query - name: filter[resource_tag_value__in] - schema: - type: array - items: - type: string - description: Multiple values may be separated by commas. - explode: false - style: form - - in: query - name: filter[resource_tags] - schema: - type: array - items: - type: string - description: |- - Filter by resource tags `key:value` pairs. - Multiple values may be separated by commas. - explode: false - style: form - in: query name: filter[resource_type] schema: @@ -1280,7 +1190,6 @@ paths: - services - regions - resource_types - - tags description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false @@ -1498,51 +1407,6 @@ paths: description: Multiple values may be separated by commas. explode: false style: form - - in: query - name: filter[resource_tag_key] - schema: - type: string - - in: query - name: filter[resource_tag_key__icontains] - schema: - type: string - - in: query - name: filter[resource_tag_key__in] - schema: - type: array - items: - type: string - description: Multiple values may be separated by commas. - explode: false - style: form - - in: query - name: filter[resource_tag_value] - schema: - type: string - - in: query - name: filter[resource_tag_value__icontains] - schema: - type: string - - in: query - name: filter[resource_tag_value__in] - schema: - type: array - items: - type: string - description: Multiple values may be separated by commas. - explode: false - style: form - - in: query - name: filter[resource_tags] - schema: - type: array - items: - type: string - description: |- - Filter by resource tags `key:value` pairs. - Multiple values may be separated by commas. - explode: false - style: form - in: query name: filter[resource_type] schema: @@ -6081,13 +5945,10 @@ components: type: array items: type: string - tags: - description: Tags are described as key-value pairs. required: - services - regions - resource_types - - tags FindingMetadataResponse: type: object properties: diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 184843af18..0173797062 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -2454,15 +2454,16 @@ class TestFindingViewSet: ("search", "ec2", 2), # full text search on finding tags ("search", "value2", 2), - ("resource_tag_key", "key", 2), - ("resource_tag_key__in", "key,key2", 2), - ("resource_tag_key__icontains", "key", 2), - ("resource_tag_value", "value", 2), - ("resource_tag_value__in", "value,value2", 2), - ("resource_tag_value__icontains", "value", 2), - ("resource_tags", "key:value", 2), - ("resource_tags", "not:exists", 0), - ("resource_tags", "not:exists,key:value", 2), + # Temporary disabled until we implement tag filtering in the UI + # ("resource_tag_key", "key", 2), + # ("resource_tag_key__in", "key,key2", 2), + # ("resource_tag_key__icontains", "key", 2), + # ("resource_tag_value", "value", 2), + # ("resource_tag_value__in", "value,value2", 2), + # ("resource_tag_value__icontains", "value", 2), + # ("resource_tags", "key:value", 2), + # ("resource_tags", "not:exists", 0), + # ("resource_tags", "not:exists,key:value", 2), ] ), ) @@ -2611,7 +2612,8 @@ class TestFindingViewSet: expected_services = {"ec2", "s3"} expected_regions = {"eu-west-1", "us-east-1"} - expected_tags = {"key": ["value"], "key2": ["value2"]} + # Temporarily disabled until we implement tag filtering in the UI + # expected_tags = {"key": ["value"], "key2": ["value2"]} expected_resource_types = {"prowler-test"} assert data["data"]["type"] == "findings-metadata" @@ -2621,7 +2623,7 @@ class TestFindingViewSet: assert ( set(data["data"]["attributes"]["resource_types"]) == expected_resource_types ) - assert data["data"]["attributes"]["tags"] == expected_tags + # assert data["data"]["attributes"]["tags"] == expected_tags def test_findings_metadata_severity_retrieve( self, authenticated_client, findings_fixture @@ -2638,7 +2640,8 @@ class TestFindingViewSet: expected_services = {"s3"} expected_regions = {"eu-west-1"} - expected_tags = {"key": ["value"], "key2": ["value2"]} + # Temporary disabled until we implement tag filtering in the UI + # expected_tags = {"key": ["value"], "key2": ["value2"]} expected_resource_types = {"prowler-test"} assert data["data"]["type"] == "findings-metadata" @@ -2648,7 +2651,7 @@ class TestFindingViewSet: assert ( set(data["data"]["attributes"]["resource_types"]) == expected_resource_types ) - assert data["data"]["attributes"]["tags"] == expected_tags + # assert data["data"]["attributes"]["tags"] == expected_tags def test_findings_metadata_future_date(self, authenticated_client): response = authenticated_client.get( @@ -2660,7 +2663,8 @@ class TestFindingViewSet: assert data["data"]["id"] is None assert data["data"]["attributes"]["services"] == [] assert data["data"]["attributes"]["regions"] == [] - assert data["data"]["attributes"]["tags"] == {} + # Temporary disabled until we implement tag filtering in the UI + # assert data["data"]["attributes"]["tags"] == {} assert data["data"]["attributes"]["resource_types"] == [] def test_findings_metadata_invalid_date(self, authenticated_client): diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 6046799d0c..4f17d2d61c 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -933,7 +933,8 @@ class FindingMetadataSerializer(serializers.Serializer): resource_types = serializers.ListField( child=serializers.CharField(), allow_empty=True ) - tags = serializers.JSONField(help_text="Tags are described as key-value pairs.") + # Temporarily disabled until we implement tag filtering in the UI + # tags = serializers.JSONField(help_text="Tags are described as key-value pairs.") class Meta: resource_name = "findings-metadata" diff --git a/api/src/backend/api/v1/urls.py b/api/src/backend/api/v1/urls.py index fd3f9ad18a..6b230960e1 100644 --- a/api/src/backend/api/v1/urls.py +++ b/api/src/backend/api/v1/urls.py @@ -1,30 +1,31 @@ +from django.conf import settings from django.urls import include, path from drf_spectacular.views import SpectacularRedocView from rest_framework_nested import routers from api.v1.views import ( + ComplianceOverviewViewSet, CustomTokenObtainView, CustomTokenRefreshView, FindingViewSet, - MembershipViewSet, - ProviderGroupViewSet, - ProviderGroupProvidersRelationshipView, - ProviderSecretViewSet, - InvitationViewSet, InvitationAcceptViewSet, - RoleViewSet, - RoleProviderGroupRelationshipView, - UserRoleRelationshipView, + InvitationViewSet, + MembershipViewSet, OverviewViewSet, - ComplianceOverviewViewSet, + ProviderGroupProvidersRelationshipView, + ProviderGroupViewSet, + ProviderSecretViewSet, ProviderViewSet, ResourceViewSet, + RoleProviderGroupRelationshipView, + RoleViewSet, ScanViewSet, ScheduleViewSet, SchemaView, TaskViewSet, TenantMembersViewSet, TenantViewSet, + UserRoleRelationshipView, UserViewSet, ) @@ -112,3 +113,6 @@ urlpatterns = [ path("schema", SchemaView.as_view(), name="schema"), path("docs", SpectacularRedocView.as_view(url_name="schema"), name="docs"), ] + +if settings.DEBUG: + urlpatterns += [path("silk/", include("silk.urls", namespace="silk"))] diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index debfcdd57c..3ecb6474fb 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -4,7 +4,7 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum -from django.db.models.functions import Coalesce, JSONObject +from django.db.models.functions import Coalesce from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control @@ -193,7 +193,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.3.1" + spectacular_settings.VERSION = "1.3.2" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -1392,48 +1392,59 @@ class FindingViewSet(BaseRLSViewSet): @action(detail=False, methods=["get"], url_name="metadata") def metadata(self, request): + tenant_id = self.request.tenant_id queryset = self.get_queryset() filtered_queryset = self.filter_queryset(queryset) - result = filtered_queryset.aggregate( - services=ArrayAgg("resources__service", flat=True, distinct=True), - regions=ArrayAgg("resources__region", flat=True, distinct=True), - tags=ArrayAgg( - JSONObject( - key=F("resources__tags__key"), value=F("resources__tags__value") - ), - distinct=True, - filter=Q(resources__tags__key__isnull=False), - ), - resource_types=ArrayAgg("resources__type", flat=True, distinct=True), - ) - if result["services"] is None: - result["services"] = [] - if result["regions"] is None: - result["regions"] = [] - if result["regions"] is None: - result["regions"] = [] - if result["resource_types"] is None: - result["resource_types"] = [] - if result["tags"] is None: - result["tags"] = [] + relevant_resources = Resource.objects.filter( + tenant_id=tenant_id, findings__in=filtered_queryset + ).distinct() - tags_dict = {} - for t in result["tags"]: - key, value = t["key"], t["value"] - if key not in tags_dict: - tags_dict[key] = [] - tags_dict[key].append(value) - - result["tags"] = tags_dict - - serializer = self.get_serializer( - data=result, + services = ( + relevant_resources.values_list("service", flat=True) + .distinct() + .order_by("service") ) + regions = ( + relevant_resources.exclude(region="") + .values_list("region", flat=True) + .distinct() + .order_by("region") + ) + + resource_types = ( + relevant_resources.values_list("type", flat=True) + .distinct() + .order_by("type") + ) + + # Temporarily disabled until we implement tag filtering in the UI + # tag_data = ( + # relevant_resources + # .filter(tags__key__isnull=False, tags__value__isnull=False) + # .exclude(tags__key="") + # .exclude(tags__value="") + # .values("tags__key", "tags__value") + # .distinct() + # .order_by("tags__key", "tags__value") + # ) + # + # tags_dict = {} + # for row in tag_data: + # k, v = row["tags__key"], row["tags__value"] + # tags_dict.setdefault(k, []).append(v) + + result = { + "services": list(services), + "regions": list(regions), + "resource_types": list(resource_types), + # "tags": tags_dict + } + + serializer = self.get_serializer(data=result) serializer.is_valid(raise_exception=True) - - return Response(data=serializer.data, status=status.HTTP_200_OK) + return Response(serializer.data, status=status.HTTP_200_OK) @extend_schema_view( diff --git a/api/src/backend/config/django/devel.py b/api/src/backend/config/django/devel.py index 825e1ce36a..6ee92a6ecb 100644 --- a/api/src/backend/config/django/devel.py +++ b/api/src/backend/config/django/devel.py @@ -37,3 +37,9 @@ REST_FRAMEWORK["DEFAULT_FILTER_BACKENDS"] = tuple( # noqa: F405 ) + ("api.filters.CustomDjangoFilterBackend",) SECRETS_ENCRYPTION_KEY = "ZMiYVo7m4Fbe2eXXPyrwxdJss2WSalXSv3xHBcJkPl0=" + +MIDDLEWARE += [ # noqa: F405 + "silk.middleware.SilkyMiddleware", +] + +INSTALLED_APPS += ["silk"] # noqa: F405 diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 25993f5bbc..89612ae6fd 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -152,6 +152,9 @@ def perform_prowler_scan( for progress, findings in prowler_scan.scan(): for finding in findings: + if finding is None: + logger.error(f"None finding detected on scan {scan_id}.") + continue for attempt in range(CELERY_DEADLOCK_ATTEMPTS): try: with rls_transaction(tenant_id): @@ -176,7 +179,10 @@ def perform_prowler_scan( # Update resource fields if necessary updated_fields = [] - if resource_instance.region != finding.region: + if ( + finding.region + and resource_instance.region != finding.region + ): resource_instance.region = finding.region updated_fields.append("region") if resource_instance.service != finding.service_name: