diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 05520c2cf3..827e9d4818 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,9 +2,17 @@ All notable changes to the **Prowler API** are documented in this file. +## [1.19.0] (Prowler UNRELEASED) + +### πŸ”„ Changed + +- Lazy-load providers and compliance data to reduce API/worker startup memory and time [(#9857)](https://github.com/prowler-cloud/prowler/pull/9857) + +--- + ## [1.18.1] (Prowler v5.17.1) -### Fixed +### 🐞 Fixed - Improve API startup process by `manage.py` argument detection [(#9856)](https://github.com/prowler-cloud/prowler/pull/9856) - Deleting providers don't try to delete a `None` Neo4j database when an Attack Paths scan is scheduled [(#9858)](https://github.com/prowler-cloud/prowler/pull/9858) @@ -13,9 +21,11 @@ All notable changes to the **Prowler API** are documented in this file. - Lazy load Neo4j driver [(#9868)](https://github.com/prowler-cloud/prowler/pull/9868) - Use `Findings.all_objects` to avoid the `ActiveProviderPartitionedManager` [(#9869)](https://github.com/prowler-cloud/prowler/pull/9869) +--- + ## [1.18.0] (Prowler v5.17.0) -### Added +### πŸš€ Added - `/api/v1/overviews/compliance-watchlist` endpoint to retrieve the compliance watchlist [(#9596)](https://github.com/prowler-cloud/prowler/pull/9596) - AlibabaCloud provider support [(#9485)](https://github.com/prowler-cloud/prowler/pull/9485) @@ -24,7 +34,7 @@ All notable changes to the **Prowler API** are documented in this file. - `provider_id` and `provider_id__in` filter aliases for findings endpoints to enable consistent frontend parameter naming [(#9701)](https://github.com/prowler-cloud/prowler/pull/9701) - Attack Paths: `/api/v1/attack-paths-scans` for AWS providers backed by Neo4j [(#9805)](https://github.com/prowler-cloud/prowler/pull/9805) -### Security +### πŸ” Security - Django 5.1.15 (CVE-2025-64460, CVE-2025-13372), Werkzeug 3.1.4 (CVE-2025-66221), sqlparse 0.5.5 (PVE-2025-82038), fonttools 4.60.2 (CVE-2025-66034) [(#9730)](https://github.com/prowler-cloud/prowler/pull/9730) - `safety` to `3.7.0` and `filelock` to `3.20.3` due to [Safety vulnerability 82754 (CVE-2025-68146)](https://data.safetycli.com/v/82754/97c/) [(#9816)](https://github.com/prowler-cloud/prowler/pull/9816) @@ -35,11 +45,11 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.17.1] (Prowler v5.16.1) -### Changed +### πŸ”„ Changed - Security Hub integration error when no regions [(#9635)](https://github.com/prowler-cloud/prowler/pull/9635) -### Fixed +### 🐞 Fixed - Orphan scheduled scans caused by transaction isolation during provider creation [(#9633)](https://github.com/prowler-cloud/prowler/pull/9633) @@ -47,19 +57,19 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.17.0] (Prowler v5.16.0) -### Added +### πŸš€ Added - New endpoint to retrieve and overview of the categories based on finding severities [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529) - Endpoints `GET /findings` and `GET /findings/latests` can now use the category filter [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529) - Account id, alias and provider name to PDF reporting table [(#9574)](https://github.com/prowler-cloud/prowler/pull/9574) -### Changed +### πŸ”„ Changed - Endpoint `GET /overviews/attack-surfaces` no longer returns the related check IDs [(#9529)](https://github.com/prowler-cloud/prowler/pull/9529) - OpenAI provider to only load chat-compatible models with tool calling support [(#9523)](https://github.com/prowler-cloud/prowler/pull/9523) - Increased execution delay for the first scheduled scan tasks to 5 seconds[(#9558)](https://github.com/prowler-cloud/prowler/pull/9558) -### Fixed +### 🐞 Fixed - Made `scan_id` a required filter in the compliance overview endpoint [(#9560)](https://github.com/prowler-cloud/prowler/pull/9560) - Reduced unnecessary UPDATE resources operations by only saving when tag mappings change, lowering write load during scans [(#9569)](https://github.com/prowler-cloud/prowler/pull/9569) @@ -68,13 +78,13 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.16.1] (Prowler v5.15.1) -### Fixed +### 🐞 Fixed - Race condition in scheduled scan creation by adding countdown to task [(#9516)](https://github.com/prowler-cloud/prowler/pull/9516) ## [1.16.0] (Prowler v5.15.0) -### Added +### πŸš€ Added - New endpoint to retrieve an overview of the attack surfaces [(#9309)](https://github.com/prowler-cloud/prowler/pull/9309) - New endpoint `GET /api/v1/overviews/findings_severity/timeseries` to retrieve daily aggregated findings by severity level [(#9363)](https://github.com/prowler-cloud/prowler/pull/9363) @@ -82,7 +92,7 @@ All notable changes to the **Prowler API** are documented in this file. - Exception handler for provider deletions during scans [(#9414)](https://github.com/prowler-cloud/prowler/pull/9414) - Support to use admin credentials through the read replica database [(#9440)](https://github.com/prowler-cloud/prowler/pull/9440) -### Changed +### πŸ”„ Changed - Error messages from Lighthouse celery tasks [(#9165)](https://github.com/prowler-cloud/prowler/pull/9165) - Restore the compliance overview endpoint's mandatory filters [(#9338)](https://github.com/prowler-cloud/prowler/pull/9338) @@ -91,7 +101,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.15.2] (Prowler v5.14.2) -### Fixed +### 🐞 Fixed - Unique constraint violation during compliance overviews task [(#9436)](https://github.com/prowler-cloud/prowler/pull/9436) - Division by zero error in ENS PDF report when all requirements are manual [(#9443)](https://github.com/prowler-cloud/prowler/pull/9443) @@ -100,7 +110,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.15.1] (Prowler v5.14.1) -### Fixed +### 🐞 Fixed - Fix typo in PDF reporting [(#9345)](https://github.com/prowler-cloud/prowler/pull/9345) - Fix IaC provider initialization failure when mutelist processor is configured [(#9331)](https://github.com/prowler-cloud/prowler/pull/9331) @@ -110,7 +120,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.15.0] (Prowler v5.14.0) -### Added +### πŸš€ Added - IaC (Infrastructure as Code) provider support for remote repositories [(#8751)](https://github.com/prowler-cloud/prowler/pull/8751) - Extend `GET /api/v1/providers` with provider-type filters and optional pagination disable to support the new Overview filters [(#8975)](https://github.com/prowler-cloud/prowler/pull/8975) @@ -130,12 +140,12 @@ All notable changes to the **Prowler API** are documented in this file. - Enhanced compliance overview endpoint with provider filtering and latest scan aggregation [(#9244)](https://github.com/prowler-cloud/prowler/pull/9244) - New endpoint `GET /api/v1/overview/regions` to retrieve aggregated findings data by region [(#9273)](https://github.com/prowler-cloud/prowler/pull/9273) -### Changed +### πŸ”„ Changed - Optimized database write queries for scan related tasks [(#9190)](https://github.com/prowler-cloud/prowler/pull/9190) - Date filters are now optional for `GET /api/v1/overviews/services` endpoint; returns latest scan data by default [(#9248)](https://github.com/prowler-cloud/prowler/pull/9248) -### Fixed +### 🐞 Fixed - Scans no longer fail when findings have UIDs exceeding 300 characters; such findings are now skipped with detailed logging [(#9246)](https://github.com/prowler-cloud/prowler/pull/9246) - Updated unique constraint for `Provider` model to exclude soft-deleted entries, resolving duplicate errors when re-deleting providers [(#9054)](https://github.com/prowler-cloud/prowler/pull/9054) @@ -144,7 +154,7 @@ All notable changes to the **Prowler API** are documented in this file. - Severity overview endpoint now ignores muted findings as expected [(#9283)](https://github.com/prowler-cloud/prowler/pull/9283) - Fixed discrepancy between ThreatScore PDF report values and database calculations [(#9296)](https://github.com/prowler-cloud/prowler/pull/9296) -### Security +### πŸ” Security - Django updated to the latest 5.1 security release, 5.1.14, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/113) and [denial-of-service vulnerability](https://github.com/prowler-cloud/prowler/security/dependabot/114) [(#9176)](https://github.com/prowler-cloud/prowler/pull/9176) @@ -152,7 +162,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.14.1] (Prowler v5.13.1) -### Fixed +### 🐞 Fixed - `/api/v1/overviews/providers` collapses data by provider type so the UI receives a single aggregated record per cloud family even when multiple accounts exist [(#9053)](https://github.com/prowler-cloud/prowler/pull/9053) - Added retry logic to database transactions to handle Aurora read replica connection failures during scale-down events [(#9064)](https://github.com/prowler-cloud/prowler/pull/9064) @@ -162,7 +172,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.14.0] (Prowler v5.13.0) -### Added +### πŸš€ Added - Default JWT keys are generated and stored if they are missing from configuration [(#8655)](https://github.com/prowler-cloud/prowler/pull/8655) - `compliance_name` for each compliance [(#7920)](https://github.com/prowler-cloud/prowler/pull/7920) @@ -176,12 +186,12 @@ All notable changes to the **Prowler API** are documented in this file. - Support Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000) - Add `provider_id__in` filter support to findings and findings severity overview endpoints [(#8951)](https://github.com/prowler-cloud/prowler/pull/8951) -### Changed +### πŸ”„ Changed - Now the MANAGE_ACCOUNT permission is required to modify or read user permissions instead of MANAGE_USERS [(#8281)](https://github.com/prowler-cloud/prowler/pull/8281) - Now at least one user with MANAGE_ACCOUNT permission is required in the tenant [(#8729)](https://github.com/prowler-cloud/prowler/pull/8729) -### Security +### πŸ” Security - Django updated to the latest 5.1 security release, 5.1.13, due to problems with potential [SQL injection](https://github.com/prowler-cloud/prowler/security/dependabot/104) and [directory traversals](https://github.com/prowler-cloud/prowler/security/dependabot/103) [(#8842)](https://github.com/prowler-cloud/prowler/pull/8842) @@ -189,7 +199,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.13.2] (Prowler v5.12.3) -### Fixed +### 🐞 Fixed - 500 error when deleting user [(#8731)](https://github.com/prowler-cloud/prowler/pull/8731) @@ -197,11 +207,11 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.13.1] (Prowler v5.12.2) -### Changed +### πŸ”„ Changed - Renamed compliance overview task queue to `compliance` [(#8755)](https://github.com/prowler-cloud/prowler/pull/8755) -### Security +### πŸ” Security - Django updated to the latest 5.1 security release, 5.1.12, due to [problems](https://www.djangoproject.com/weblog/2025/sep/03/security-releases/) with potential SQL injection in FilteredRelation column aliases [(#8693)](https://github.com/prowler-cloud/prowler/pull/8693) @@ -209,7 +219,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.13.0] (Prowler v5.12.0) -### Added +### πŸš€ Added - Integration with JIRA, enabling sending findings to a JIRA project [(#8622)](https://github.com/prowler-cloud/prowler/pull/8622), [(#8637)](https://github.com/prowler-cloud/prowler/pull/8637) - `GET /overviews/findings_severity` now supports `filter[status]` and `filter[status__in]` to aggregate by specific statuses (`FAIL`, `PASS`)[(#8186)](https://github.com/prowler-cloud/prowler/pull/8186) @@ -219,13 +229,13 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.12.0] (Prowler v5.11.0) -### Added +### πŸš€ Added - Lighthouse support for OpenAI GPT-5 [(#8527)](https://github.com/prowler-cloud/prowler/pull/8527) - Integration with Amazon Security Hub, enabling sending findings to Security Hub [(#8365)](https://github.com/prowler-cloud/prowler/pull/8365) - Generate ASFF output for AWS providers with SecurityHub integration enabled [(#8569)](https://github.com/prowler-cloud/prowler/pull/8569) -### Fixed +### 🐞 Fixed - GitHub provider always scans user instead of organization when using provider UID [(#8587)](https://github.com/prowler-cloud/prowler/pull/8587) @@ -233,12 +243,12 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.11.0] (Prowler v5.10.0) -### Added +### πŸš€ Added - Github provider support [(#8271)](https://github.com/prowler-cloud/prowler/pull/8271) - Integration with Amazon S3, enabling storage and retrieval of scan data via S3 buckets [(#8056)](https://github.com/prowler-cloud/prowler/pull/8056) -### Fixed +### 🐞 Fixed - Avoid sending errors to Sentry in M365 provider when user authentication fails [(#8420)](https://github.com/prowler-cloud/prowler/pull/8420) @@ -246,7 +256,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [1.10.2] (Prowler v5.9.2) -### Changed +### πŸ”„ Changed - Optimized queries for resources views [(#8336)](https://github.com/prowler-cloud/prowler/pull/8336) @@ -254,7 +264,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.10.1] (Prowler v5.9.1) -### Fixed +### 🐞 Fixed - Calculate failed findings during scans to prevent heavy database queries [(#8322)](https://github.com/prowler-cloud/prowler/pull/8322) @@ -262,28 +272,28 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.10.0] (Prowler v5.9.0) -### Added +### πŸš€ Added - SSO with SAML support [(#8175)](https://github.com/prowler-cloud/prowler/pull/8175) - `GET /resources/metadata`, `GET /resources/metadata/latest` and `GET /resources/latest` to expose resource metadata and latest scan results [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112) -### Changed +### πŸ”„ Changed - `/processors` endpoints to post-process findings. Currently, only the Mutelist processor is supported to allow to mute findings. - Optimized the underlying queries for resources endpoints [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112) - Optimized include parameters for resources view [(#8229)](https://github.com/prowler-cloud/prowler/pull/8229) - Optimized overview background tasks [(#8300)](https://github.com/prowler-cloud/prowler/pull/8300) -### Fixed +### 🐞 Fixed - Search filter for findings and resources [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112) - RBAC is now applied to `GET /overviews/providers` [(#8277)](https://github.com/prowler-cloud/prowler/pull/8277) -### Changed +### πŸ”„ Changed - `POST /schedules/daily` returns a `409 CONFLICT` if already created [(#8258)](https://github.com/prowler-cloud/prowler/pull/8258) -### Security +### πŸ” Security - Enhanced password validation to enforce 12+ character passwords with special characters, uppercase, lowercase, and numbers [(#8225)](https://github.com/prowler-cloud/prowler/pull/8225) @@ -291,20 +301,20 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.9.1] (Prowler v5.8.1) -### Added +### πŸš€ Added - Custom exception for provider connection errors during scans [(#8234)](https://github.com/prowler-cloud/prowler/pull/8234) -### Changed +### πŸ”„ Changed - Summary and overview tasks now use a dedicated queue and no longer propagate errors to compliance tasks [(#8214)](https://github.com/prowler-cloud/prowler/pull/8214) -### Fixed +### 🐞 Fixed - Scan with no resources will not trigger legacy code for findings metadata [(#8183)](https://github.com/prowler-cloud/prowler/pull/8183) - Invitation email comparison case-insensitive [(#8206)](https://github.com/prowler-cloud/prowler/pull/8206) -### Removed +### ❌ Removed - Validation of the provider's secret type during updates [(#8197)](https://github.com/prowler-cloud/prowler/pull/8197) @@ -312,18 +322,18 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.9.0] (Prowler v5.8.0) -### Added +### πŸš€ Added - Support GCP Service Account key [(#7824)](https://github.com/prowler-cloud/prowler/pull/7824) - `GET /compliance-overviews` endpoints to retrieve compliance metadata and specific requirements statuses [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877) - Lighthouse configuration support [(#7848)](https://github.com/prowler-cloud/prowler/pull/7848) -### Changed +### πŸ”„ Changed - Reworked `GET /compliance-overviews` to return proper requirement metrics [(#7877)](https://github.com/prowler-cloud/prowler/pull/7877) - Optional `user` and `password` for M365 provider [(#7992)](https://github.com/prowler-cloud/prowler/pull/7992) -### Fixed +### 🐞 Fixed - Scheduled scans are no longer deleted when their daily schedule run is disabled [(#8082)](https://github.com/prowler-cloud/prowler/pull/8082) @@ -331,7 +341,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.8.5] (Prowler v5.7.5) -### Fixed +### 🐞 Fixed - Normalize provider UID to ensure safe and unique export directory paths [(#8007)](https://github.com/prowler-cloud/prowler/pull/8007). - Blank resource types in `/metadata` endpoints [(#8027)](https://github.com/prowler-cloud/prowler/pull/8027) @@ -340,7 +350,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.8.4] (Prowler v5.7.4) -### Removed +### ❌ Removed - Reverted RLS transaction handling and DB custom backend [(#7994)](https://github.com/prowler-cloud/prowler/pull/7994) @@ -348,15 +358,15 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.8.3] (Prowler v5.7.3) -### Added +### πŸš€ Added - Database backend to handle already closed connections [(#7935)](https://github.com/prowler-cloud/prowler/pull/7935) -### Changed +### πŸ”„ Changed - Renamed field encrypted_password to password for M365 provider [(#7784)](https://github.com/prowler-cloud/prowler/pull/7784) -### Fixed +### 🐞 Fixed - Transaction persistence with RLS operations [(#7916)](https://github.com/prowler-cloud/prowler/pull/7916) - Reverted the change `get_with_retry` to use the original `get` method for retrieving tasks [(#7932)](https://github.com/prowler-cloud/prowler/pull/7932) @@ -365,7 +375,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.8.2] (Prowler v5.7.2) -### Fixed +### 🐞 Fixed - Task lookup to use task_kwargs instead of task_args for scan report resolution [(#7830)](https://github.com/prowler-cloud/prowler/pull/7830) - Kubernetes UID validation to allow valid context names [(#7871)](https://github.com/prowler-cloud/prowler/pull/7871) @@ -377,7 +387,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.8.1] (Prowler v5.7.1) -### Fixed +### 🐞 Fixed - Added database index to improve performance on finding lookup [(#7800)](https://github.com/prowler-cloud/prowler/pull/7800) @@ -385,7 +395,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.8.0] (Prowler v5.7.0) -### Added +### πŸš€ Added - Huge improvements to `/findings/metadata` and resource related filters for findings [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690) - Improvements to `/overviews` endpoints [(#7690)](https://github.com/prowler-cloud/prowler/pull/7690) @@ -397,7 +407,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.7.0] (Prowler v5.6.0) -### Added +### πŸš€ Added - M365 as a new provider [(#7563)](https://github.com/prowler-cloud/prowler/pull/7563) - `compliance/` folder and ZIP‐export functionality for all compliance reports [(#7653)](https://github.com/prowler-cloud/prowler/pull/7653) @@ -407,7 +417,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.6.0] (Prowler v5.5.0) -### Added +### πŸš€ Added - Support for developing new integrations [(#7167)](https://github.com/prowler-cloud/prowler/pull/7167) - HTTP Security Headers [(#7289)](https://github.com/prowler-cloud/prowler/pull/7289) @@ -419,7 +429,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.5.4] (Prowler v5.4.4) -### Fixed +### 🐞 Fixed - Bug with periodic tasks when trying to delete a provider [(#7466)](https://github.com/prowler-cloud/prowler/pull/7466) @@ -427,7 +437,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.5.3] (Prowler v5.4.3) -### Fixed +### 🐞 Fixed - Duplicated scheduled scans handling [(#7401)](https://github.com/prowler-cloud/prowler/pull/7401) - Environment variable to configure the deletion task batch size [(#7423)](https://github.com/prowler-cloud/prowler/pull/7423) @@ -436,7 +446,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.5.2] (Prowler v5.4.2) -### Changed +### πŸ”„ Changed - Refactored deletion logic and implemented retry mechanism for deletion tasks [(#7349)](https://github.com/prowler-cloud/prowler/pull/7349) @@ -444,7 +454,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.5.1] (Prowler v5.4.1) -### Fixed +### 🐞 Fixed - Handle response in case local files are missing [(#7183)](https://github.com/prowler-cloud/prowler/pull/7183) - Race condition when deleting export files after the S3 upload [(#7172)](https://github.com/prowler-cloud/prowler/pull/7172) @@ -454,13 +464,13 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.5.0] (Prowler v5.4.0) -### Added +### πŸš€ Added - Social login integration with Google and GitHub [(#6906)](https://github.com/prowler-cloud/prowler/pull/6906) - API scan report system, now all scans launched from the API will generate a compressed file with the report in OCSF, CSV and HTML formats [(#6878)](https://github.com/prowler-cloud/prowler/pull/6878) - Configurable Sentry integration [(#6874)](https://github.com/prowler-cloud/prowler/pull/6874) -### Changed +### πŸ”„ Changed - Optimized `GET /findings` endpoint to improve response time and size [(#7019)](https://github.com/prowler-cloud/prowler/pull/7019) @@ -468,7 +478,7 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.4.0] (Prowler v5.3.0) -### Changed +### πŸ”„ Changed - Daily scheduled scan instances are now created beforehand with `SCHEDULED` state [(#6700)](https://github.com/prowler-cloud/prowler/pull/6700) - Findings endpoints now require at least one date filter [(#6800)](https://github.com/prowler-cloud/prowler/pull/6800) diff --git a/api/src/backend/api/apps.py b/api/src/backend/api/apps.py index a690e501a3..340981af42 100644 --- a/api/src/backend/api/apps.py +++ b/api/src/backend/api/apps.py @@ -30,7 +30,6 @@ class ApiConfig(AppConfig): def ready(self): from api import schema_extensions # noqa: F401 from api import signals # noqa: F401 - from api.compliance import load_prowler_compliance # Generate required cryptographic keys if not present, but only if: # `"manage.py" not in sys.argv[0]`: If an external server (e.g., Gunicorn) is running the app @@ -44,8 +43,6 @@ class ApiConfig(AppConfig): # Neo4j driver is initialized lazily on first use (see api.attack_paths.database) # This avoids connection attempts during regular scans that don't need graph database - load_prowler_compliance() - def _ensure_crypto_keys(self): """ Orchestrator method that ensures all required cryptographic keys are present. diff --git a/api/src/backend/api/compliance.py b/api/src/backend/api/compliance.py index da39fc23bb..1705ed2e8f 100644 --- a/api/src/backend/api/compliance.py +++ b/api/src/backend/api/compliance.py @@ -1,15 +1,99 @@ -from types import MappingProxyType +from collections.abc import Iterable, Mapping from api.models import Provider from prowler.config.config import get_available_compliance_frameworks from prowler.lib.check.compliance_models import Compliance from prowler.lib.check.models import CheckMetadata -PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = {} -PROWLER_CHECKS = {} AVAILABLE_COMPLIANCE_FRAMEWORKS = {} +class LazyComplianceTemplate(Mapping): + """Lazy-load compliance templates per provider on first access.""" + + def __init__(self, provider_types: Iterable[str] | None = None) -> None: + if provider_types is None: + provider_types = Provider.ProviderChoices.values + self._provider_types = tuple(provider_types) + self._provider_types_set = set(self._provider_types) + self._cache: dict[str, dict] = {} + + def _load_provider(self, provider_type: str) -> dict: + if provider_type not in self._provider_types_set: + raise KeyError(provider_type) + cached = self._cache.get(provider_type) + if cached is not None: + return cached + _ensure_provider_loaded(provider_type) + return self._cache[provider_type] + + def __getitem__(self, key: str) -> dict: + return self._load_provider(key) + + def __iter__(self): + return iter(self._provider_types) + + def __len__(self) -> int: + return len(self._provider_types) + + def __contains__(self, key: object) -> bool: + return key in self._provider_types_set + + def get(self, key: str, default=None): + if key not in self._provider_types_set: + return default + return self._load_provider(key) + + def __repr__(self) -> str: # pragma: no cover - debugging helper + loaded = ", ".join(sorted(self._cache)) + return f"{self.__class__.__name__}(loaded=[{loaded}])" + + +class LazyChecksMapping(Mapping): + """Lazy-load checks mapping per provider on first access.""" + + def __init__(self, provider_types: Iterable[str] | None = None) -> None: + if provider_types is None: + provider_types = Provider.ProviderChoices.values + self._provider_types = tuple(provider_types) + self._provider_types_set = set(self._provider_types) + self._cache: dict[str, dict] = {} + + def _load_provider(self, provider_type: str) -> dict: + if provider_type not in self._provider_types_set: + raise KeyError(provider_type) + cached = self._cache.get(provider_type) + if cached is not None: + return cached + _ensure_provider_loaded(provider_type) + return self._cache[provider_type] + + def __getitem__(self, key: str) -> dict: + return self._load_provider(key) + + def __iter__(self): + return iter(self._provider_types) + + def __len__(self) -> int: + return len(self._provider_types) + + def __contains__(self, key: object) -> bool: + return key in self._provider_types_set + + def get(self, key: str, default=None): + if key not in self._provider_types_set: + return default + return self._load_provider(key) + + def __repr__(self) -> str: # pragma: no cover - debugging helper + loaded = ", ".join(sorted(self._cache)) + return f"{self.__class__.__name__}(loaded=[{loaded}])" + + +PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = LazyComplianceTemplate() +PROWLER_CHECKS = LazyChecksMapping() + + def get_compliance_frameworks(provider_type: Provider.ProviderChoices) -> list[str]: """ Retrieve and cache the list of available compliance frameworks for a specific cloud provider. @@ -70,28 +154,35 @@ def get_prowler_provider_compliance(provider_type: Provider.ProviderChoices) -> return Compliance.get_bulk(provider_type) -def load_prowler_compliance(): - """ - Load and initialize the Prowler compliance data and checks for all provider types. - - This function retrieves compliance data for all supported provider types, - generates a compliance overview template, and populates the global variables - `PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE` and `PROWLER_CHECKS` with read-only mappings - of the compliance templates and checks, respectively. - """ - global PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE - global PROWLER_CHECKS - - prowler_compliance = { - provider_type: get_prowler_provider_compliance(provider_type) - for provider_type in Provider.ProviderChoices.values - } - template = generate_compliance_overview_template(prowler_compliance) - PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE = MappingProxyType(template) - PROWLER_CHECKS = MappingProxyType(load_prowler_checks(prowler_compliance)) +def _load_provider_assets(provider_type: Provider.ProviderChoices) -> tuple[dict, dict]: + prowler_compliance = {provider_type: get_prowler_provider_compliance(provider_type)} + template = generate_compliance_overview_template( + prowler_compliance, provider_types=[provider_type] + ) + checks = load_prowler_checks(prowler_compliance, provider_types=[provider_type]) + return template.get(provider_type, {}), checks.get(provider_type, {}) -def load_prowler_checks(prowler_compliance): +def _ensure_provider_loaded(provider_type: Provider.ProviderChoices) -> None: + if ( + provider_type in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache + and provider_type in PROWLER_CHECKS._cache + ): + return + template_cached = PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache.get(provider_type) + checks_cached = PROWLER_CHECKS._cache.get(provider_type) + if template_cached is not None and checks_cached is not None: + return + template, checks = _load_provider_assets(provider_type) + if template_cached is None: + PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE._cache[provider_type] = template + if checks_cached is None: + PROWLER_CHECKS._cache[provider_type] = checks + + +def load_prowler_checks( + prowler_compliance, provider_types: Iterable[str] | None = None +): """ Generate a mapping of checks to the compliance frameworks that include them. @@ -100,21 +191,25 @@ def load_prowler_checks(prowler_compliance): of compliance names that include that check. Args: - prowler_compliance (dict): The compliance data for all provider types, + prowler_compliance (dict): The compliance data for provider types, as returned by `get_prowler_provider_compliance`. + provider_types (Iterable[str] | None): Optional subset of provider types to + process. Defaults to all providers. Returns: dict: A nested dictionary where the first-level keys are provider types, and the values are dictionaries mapping check IDs to sets of compliance names. """ checks = {} - for provider_type in Provider.ProviderChoices.values: + if provider_types is None: + provider_types = Provider.ProviderChoices.values + for provider_type in provider_types: checks[provider_type] = { check_id: set() for check_id in get_prowler_provider_checks(provider_type) } - for compliance_name, compliance_data in prowler_compliance[ - provider_type - ].items(): + for compliance_name, compliance_data in prowler_compliance.get( + provider_type, {} + ).items(): for requirement in compliance_data.Requirements: for check in requirement.Checks: try: @@ -163,7 +258,9 @@ def generate_scan_compliance( ] += 1 -def generate_compliance_overview_template(prowler_compliance: dict): +def generate_compliance_overview_template( + prowler_compliance: dict, provider_types: Iterable[str] | None = None +): """ Generate a compliance overview template for all provider types. @@ -173,17 +270,21 @@ def generate_compliance_overview_template(prowler_compliance: dict): counts for requirements status. Args: - prowler_compliance (dict): The compliance data for all provider types, + prowler_compliance (dict): The compliance data for provider types, as returned by `get_prowler_provider_compliance`. + provider_types (Iterable[str] | None): Optional subset of provider types to + process. Defaults to all providers. Returns: dict: A nested dictionary representing the compliance overview template, structured by provider type and compliance framework. """ template = {} - for provider_type in Provider.ProviderChoices.values: + if provider_types is None: + provider_types = Provider.ProviderChoices.values + for provider_type in provider_types: provider_compliance = template.setdefault(provider_type, {}) - compliance_data_dict = prowler_compliance[provider_type] + compliance_data_dict = prowler_compliance.get(provider_type, {}) for compliance_name, compliance_data in compliance_data_dict.items(): compliance_requirements = {} diff --git a/api/src/backend/api/tests/test_compliance.py b/api/src/backend/api/tests/test_compliance.py index 7312335921..8774f33787 100644 --- a/api/src/backend/api/tests/test_compliance.py +++ b/api/src/backend/api/tests/test_compliance.py @@ -6,7 +6,6 @@ from api.compliance import ( get_prowler_provider_checks, get_prowler_provider_compliance, load_prowler_checks, - load_prowler_compliance, ) from api.models import Provider @@ -35,55 +34,6 @@ class TestCompliance: assert compliance_data == mock_compliance.get_bulk.return_value mock_compliance.get_bulk.assert_called_once_with(provider_type) - @patch("api.models.Provider.ProviderChoices") - @patch("api.compliance.get_prowler_provider_compliance") - @patch("api.compliance.generate_compliance_overview_template") - @patch("api.compliance.load_prowler_checks") - def test_load_prowler_compliance( - self, - mock_load_prowler_checks, - mock_generate_compliance_overview_template, - mock_get_prowler_provider_compliance, - mock_provider_choices, - ): - mock_provider_choices.values = ["aws", "azure"] - - compliance_data_aws = {"compliance_aws": MagicMock()} - compliance_data_azure = {"compliance_azure": MagicMock()} - - compliance_data_dict = { - "aws": compliance_data_aws, - "azure": compliance_data_azure, - } - - def mock_get_compliance(provider_type): - return compliance_data_dict[provider_type] - - mock_get_prowler_provider_compliance.side_effect = mock_get_compliance - - mock_generate_compliance_overview_template.return_value = { - "template_key": "template_value" - } - - mock_load_prowler_checks.return_value = {"checks_key": "checks_value"} - - load_prowler_compliance() - - from api.compliance import PROWLER_CHECKS, PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE - - assert PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE == { - "template_key": "template_value" - } - assert PROWLER_CHECKS == {"checks_key": "checks_value"} - - expected_prowler_compliance = compliance_data_dict - mock_get_prowler_provider_compliance.assert_any_call("aws") - mock_get_prowler_provider_compliance.assert_any_call("azure") - mock_generate_compliance_overview_template.assert_called_once_with( - expected_prowler_compliance - ) - mock_load_prowler_checks.assert_called_once_with(expected_prowler_compliance) - @patch("api.compliance.get_prowler_provider_checks") @patch("api.models.Provider.ProviderChoices") def test_load_prowler_checks( diff --git a/api/src/backend/api/utils.py b/api/src/backend/api/utils.py index e624a36aab..ca3e4db86b 100644 --- a/api/src/backend/api/utils.py +++ b/api/src/backend/api/utils.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from datetime import datetime, timezone +from typing import TYPE_CHECKING from allauth.socialaccount.providers.oauth2.client import OAuth2Client from django.contrib.postgres.aggregates import ArrayAgg @@ -11,19 +14,25 @@ from api.exceptions import InvitationTokenExpiredException from api.models import Integration, Invitation, Processor, Provider, Resource from api.v1.serializers import FindingMetadataSerializer from prowler.lib.outputs.jira.jira import Jira, JiraBasicAuthError -from prowler.providers.alibabacloud.alibabacloud_provider import AlibabacloudProvider -from prowler.providers.aws.aws_provider import AwsProvider from prowler.providers.aws.lib.s3.s3 import S3 from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub -from prowler.providers.azure.azure_provider import AzureProvider from prowler.providers.common.models import Connection -from prowler.providers.gcp.gcp_provider import GcpProvider -from prowler.providers.github.github_provider import GithubProvider -from prowler.providers.iac.iac_provider import IacProvider -from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider -from prowler.providers.m365.m365_provider import M365Provider -from prowler.providers.mongodbatlas.mongodbatlas_provider import MongodbatlasProvider -from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider + +if TYPE_CHECKING: + from prowler.providers.alibabacloud.alibabacloud_provider import ( + AlibabacloudProvider, + ) + from prowler.providers.aws.aws_provider import AwsProvider + from prowler.providers.azure.azure_provider import AzureProvider + from prowler.providers.gcp.gcp_provider import GcpProvider + from prowler.providers.github.github_provider import GithubProvider + from prowler.providers.iac.iac_provider import IacProvider + from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider + from prowler.providers.m365.m365_provider import M365Provider + from prowler.providers.mongodbatlas.mongodbatlas_provider import ( + MongodbatlasProvider, + ) + from prowler.providers.oraclecloud.oraclecloud_provider import OraclecloudProvider class CustomOAuth2Client(OAuth2Client): @@ -89,24 +98,52 @@ def return_prowler_provider( """ match provider.provider: case Provider.ProviderChoices.AWS.value: + from prowler.providers.aws.aws_provider import AwsProvider + prowler_provider = AwsProvider case Provider.ProviderChoices.GCP.value: + from prowler.providers.gcp.gcp_provider import GcpProvider + prowler_provider = GcpProvider case Provider.ProviderChoices.AZURE.value: + from prowler.providers.azure.azure_provider import AzureProvider + prowler_provider = AzureProvider case Provider.ProviderChoices.KUBERNETES.value: + from prowler.providers.kubernetes.kubernetes_provider import ( + KubernetesProvider, + ) + prowler_provider = KubernetesProvider case Provider.ProviderChoices.M365.value: + from prowler.providers.m365.m365_provider import M365Provider + prowler_provider = M365Provider case Provider.ProviderChoices.GITHUB.value: + from prowler.providers.github.github_provider import GithubProvider + prowler_provider = GithubProvider case Provider.ProviderChoices.MONGODBATLAS.value: + from prowler.providers.mongodbatlas.mongodbatlas_provider import ( + MongodbatlasProvider, + ) + prowler_provider = MongodbatlasProvider case Provider.ProviderChoices.IAC.value: + from prowler.providers.iac.iac_provider import IacProvider + prowler_provider = IacProvider case Provider.ProviderChoices.ORACLECLOUD.value: + from prowler.providers.oraclecloud.oraclecloud_provider import ( + OraclecloudProvider, + ) + prowler_provider = OraclecloudProvider case Provider.ProviderChoices.ALIBABACLOUD.value: + from prowler.providers.alibabacloud.alibabacloud_provider import ( + AlibabacloudProvider, + ) + prowler_provider = AlibabacloudProvider case _: raise ValueError(f"Provider type {provider.provider} not supported") diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 54d989e375..994e4b20c6 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -3,7 +3,6 @@ import glob import json import logging import os - from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta, timezone @@ -11,7 +10,6 @@ from decimal import ROUND_HALF_UP, Decimal, InvalidOperation from urllib.parse import urljoin import sentry_sdk - from allauth.socialaccount.models import SocialAccount, SocialApp from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter @@ -75,12 +73,26 @@ from rest_framework.generics import GenericAPIView, get_object_or_404 from rest_framework.permissions import SAFE_METHODS from rest_framework_json_api.views import RelationshipView, Response from rest_framework_simplejwt.exceptions import InvalidToken, TokenError - -from api.attack_paths import ( - get_queries_for_provider, - get_query_by_id, - views_helpers as attack_paths_views_helpers, +from tasks.beat import schedule_provider_scan +from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils +from tasks.jobs.export import get_s3_client +from tasks.tasks import ( + backfill_compliance_summaries_task, + backfill_scan_resource_summaries_task, + check_integration_connection_task, + check_lighthouse_connection_task, + check_lighthouse_provider_connection_task, + check_provider_connection_task, + delete_provider_task, + delete_tenant_task, + jira_integration_task, + mute_historical_findings_task, + perform_scan_task, + refresh_lighthouse_provider_models_task, ) + +from api.attack_paths import get_queries_for_provider, get_query_by_id +from api.attack_paths import views_helpers as attack_paths_views_helpers from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset from api.compliance import ( PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, @@ -90,6 +102,7 @@ from api.db_router import MainRouter from api.db_utils import rls_transaction from api.exceptions import TaskFailedException from api.filters import ( + AttackPathsScanFilter, AttackSurfaceOverviewFilter, CategoryOverviewFilter, ComplianceOverviewFilter, @@ -102,7 +115,6 @@ from api.filters import ( InvitationFilter, LatestFindingFilter, LatestResourceFilter, - AttackPathsScanFilter, LighthouseProviderConfigFilter, LighthouseProviderModelsFilter, MembershipFilter, @@ -124,6 +136,7 @@ from api.filters import ( UserFilter, ) from api.models import ( + AttackPathsScan, AttackSurfaceOverview, ComplianceOverviewSummary, ComplianceRequirementOverview, @@ -131,7 +144,6 @@ from api.models import ( Finding, Integration, Invitation, - AttackPathsScan, LighthouseConfiguration, LighthouseProviderConfiguration, LighthouseProviderModels, @@ -177,9 +189,9 @@ from api.utils import ( from api.uuid_utils import datetime_to_uuid7, uuid7_start from api.v1.mixins import DisablePaginationMixin, PaginateByPkMixin, TaskManagementMixin from api.v1.serializers import ( + AttackPathsQueryResultSerializer, AttackPathsQueryRunRequestSerializer, AttackPathsQuerySerializer, - AttackPathsQueryResultSerializer, AttackPathsScanSerializer, AttackSurfaceOverviewSerializer, CategoryOverviewSerializer, @@ -263,23 +275,6 @@ from api.v1.serializers import ( UserSerializer, UserUpdateSerializer, ) -from tasks.beat import schedule_provider_scan -from tasks.jobs.attack_paths import db_utils as attack_paths_db_utils -from tasks.jobs.export import get_s3_client -from tasks.tasks import ( - backfill_compliance_summaries_task, - backfill_scan_resource_summaries_task, - check_integration_connection_task, - check_lighthouse_connection_task, - check_lighthouse_provider_connection_task, - check_provider_connection_task, - delete_provider_task, - delete_tenant_task, - jira_integration_task, - mute_historical_findings_task, - perform_scan_task, - refresh_lighthouse_provider_models_task, -) logger = logging.getLogger(BackendLogger.API) @@ -4199,7 +4194,7 @@ class ComplianceOverviewViewSet(BaseRLSViewSet, TaskManagementMixin): # If we couldn't determine from database, try each provider type if not provider_type: for pt in Provider.ProviderChoices.values: - if compliance_id in PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE.get(pt, {}): + if compliance_id in get_compliance_frameworks(pt): provider_type = pt break diff --git a/api/src/backend/tasks/tests/test_integrations.py b/api/src/backend/tasks/tests/test_integrations.py index d37b27e320..954c645998 100644 --- a/api/src/backend/tasks/tests/test_integrations.py +++ b/api/src/backend/tasks/tests/test_integrations.py @@ -417,9 +417,8 @@ class TestProwlerIntegrationConnectionTest: raise_on_exception=False, ) - @patch("api.utils.AwsProvider") @patch("api.utils.S3") - def test_s3_integration_connection_failure(self, mock_s3_class, mock_aws_provider): + def test_s3_integration_connection_failure(self, mock_s3_class): """Test S3 integration connection failure.""" integration = MagicMock() integration.integration_type = Integration.IntegrationChoices.AMAZON_S3 @@ -429,9 +428,6 @@ class TestProwlerIntegrationConnectionTest: } integration.configuration = {"bucket_name": "test-bucket"} - mock_session = MagicMock() - mock_aws_provider.return_value.session.current_session = mock_session - mock_connection = Connection( is_connected=False, error=Exception("Bucket not found") )