diff --git a/.env b/.env index f667ad6183..e97f578170 100644 --- a/.env +++ b/.env @@ -31,6 +31,13 @@ VALKEY_PORT=6379 VALKEY_DB=0 # API scan settings + +# The path to the directory where scan artifacts should be stored +DJANGO_TMP_OUTPUT_DIRECTORY = "/tmp/prowler_api_output" + +# The maximum number of findings to process in a single batch +DJANGO_FINDINGS_BATCH_SIZE = 1000 + # The AWS access key to be used when uploading scan artifacts to an S3 bucket # If left empty, default AWS credentials resolution behavior will be used DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID="" diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index a6438c8359..ac5b6be441 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -2182,7 +2182,10 @@ class TestScanViewSet: with patch("api.v1.views.get_s3_client", side_effect=client_err): response = authenticated_client.get(url) assert response.status_code == status.HTTP_403_FORBIDDEN - assert response.data["detail"] == "There is a problem with the AWS credentials." + assert ( + response.data["detail"] + == "There is a problem with the AWS credentials." + ) def test_report_s3_get_object_fail(self, authenticated_client, scans_fixture): """ @@ -2194,8 +2197,9 @@ class TestScanViewSet: scan.save() url = reverse("scan-report", kwargs={"pk": scan.pk}) client_err = ClientError({"Error": {"Code": "NoSuchKey"}}, "get_object") - with patch("api.v1.views.get_s3_client") as mock_get_s3_client, \ - patch("api.v1.views.env.str", return_value="bucket"): + with patch("api.v1.views.get_s3_client") as mock_get_s3_client, patch( + "api.v1.views.env.str", return_value="bucket" + ): s3_client = mock_get_s3_client.return_value s3_client.get_object.side_effect = client_err response = authenticated_client.get(url) @@ -2215,8 +2219,9 @@ class TestScanViewSet: # Create a dummy S3 object with a 'Body' that returns our fake file content dummy_body = BytesIO(fake_file_content) dummy_s3_object = {"Body": dummy_body} - with patch("api.v1.views.get_s3_client") as mock_get_s3_client, \ - patch("api.v1.views.env.str", return_value="bucket"): + with patch("api.v1.views.get_s3_client") as mock_get_s3_client, patch( + "api.v1.views.env.str", return_value="bucket" + ): s3_client = mock_get_s3_client.return_value s3_client.get_object.return_value = dummy_s3_object response = authenticated_client.get(url) @@ -2252,8 +2257,9 @@ class TestScanViewSet: scan.output_path = "/path/to/file.zip" scan.save() url = reverse("scan-report", kwargs={"pk": scan.pk}) - with patch("api.v1.views.glob.glob", return_value=["/path/to/file.zip"]), \ - patch("api.v1.views.open", side_effect=IOError): + with patch("api.v1.views.glob.glob", return_value=["/path/to/file.zip"]), patch( + "api.v1.views.open", side_effect=IOError + ): response = authenticated_client.get(url) assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR assert response.data["detail"] == "Error reading local file" @@ -2269,8 +2275,9 @@ class TestScanViewSet: scan.save() url = reverse("scan-report", kwargs={"pk": scan.pk}) m_open = mock_open(read_data=fake_file_content) - with patch("api.v1.views.glob.glob", return_value=["/path/to/file.zip"]), \ - patch("api.v1.views.open", m_open): + with patch("api.v1.views.glob.glob", return_value=["/path/to/file.zip"]), patch( + "api.v1.views.open", m_open + ): response = authenticated_client.get(url) assert isinstance(response, HttpResponse) assert response.status_code == 200 diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 4d5cc6d59f..4a1e126937 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1,13 +1,11 @@ import glob import os -import boto3 +from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter +from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError from celery.result import AsyncResult from config.env import env -from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter -from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter -from celery.result import AsyncResult from config.settings.social_login import ( GITHUB_OAUTH_CALLBACK_URL, GOOGLE_OAUTH_CALLBACK_URL, @@ -18,8 +16,8 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum -from django.http import HttpResponse from django.db.models.functions import Coalesce +from django.http import HttpResponse from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control @@ -46,6 +44,7 @@ from rest_framework.permissions import SAFE_METHODS from rest_framework_json_api.views import RelationshipView, Response from rest_framework_simplejwt.exceptions import InvalidToken, TokenError from tasks.beat import schedule_provider_scan +from tasks.jobs.export import get_s3_client from tasks.tasks import ( check_provider_connection_task, delete_provider_task, @@ -95,7 +94,6 @@ from api.models import ( from api.pagination import ComplianceOverviewPagination from api.rbac.permissions import Permissions, get_providers, get_role from api.rls import Tenant -from tasks.jobs.export import get_s3_client from api.utils import CustomOAuth2Client, validate_invitation from api.uuid_utils import datetime_to_uuid7 from api.v1.serializers import ( diff --git a/api/src/backend/config/django/base.py b/api/src/backend/config/django/base.py index 8641a09af1..271599844e 100644 --- a/api/src/backend/config/django/base.py +++ b/api/src/backend/config/django/base.py @@ -222,11 +222,17 @@ FINDINGS_MAX_DAYS_IN_RANGE = env.int("DJANGO_FINDINGS_MAX_DAYS_IN_RANGE", 7) # API export settings -DJANGO_TMP_OUTPUT_DIRECTORY = "/tmp/prowler_api_output" -DJANGO_FINDINGS_BATCH_SIZE = 1000 +DJANGO_TMP_OUTPUT_DIRECTORY = env.str( + "DJANGO_TMP_OUTPUT_DIRECTORY", "/tmp/prowler_api_output" +) +DJANGO_FINDINGS_BATCH_SIZE = env.str("DJANGO_FINDINGS_BATCH_SIZE", 1000) -DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET = env.str("DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET", "") +DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET = env.str( + "DJANGO_ARTIFACTS_AWS_S3_OUTPUT_BUCKET", "" +) DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID = env.str("DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID", "") -DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY = env.str("DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY", "") +DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY = env.str( + "DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY", "" +) DJANGO_ARTIFACTS_AWS_SESSION_TOKEN = env.str("DJANGO_ARTIFACTS_AWS_SESSION_TOKEN", "") DJANGO_ARTIFACTS_AWS_DEFAULT_REGION = env.str("DJANGO_ARTIFACTS_AWS_DEFAULT_REGION", "") diff --git a/api/src/backend/tasks/jobs/export.py b/api/src/backend/tasks/jobs/export.py index 0e4009b2a6..7bfa9d968d 100644 --- a/api/src/backend/tasks/jobs/export.py +++ b/api/src/backend/tasks/jobs/export.py @@ -75,9 +75,7 @@ def get_s3_client(): s3_client = boto3.client( "s3", aws_access_key_id=env.str("DJANGO_ARTIFACTS_AWS_ACCESS_KEY_ID"), - aws_secret_access_key=env.str( - "DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY" - ), + aws_secret_access_key=env.str("DJANGO_ARTIFACTS_AWS_SECRET_ACCESS_KEY"), aws_session_token=env.str("DJANGO_ARTIFACTS_AWS_SESSION_TOKEN"), region_name=env.str("DJANGO_ARTIFACTS_AWS_DEFAULT_REGION"), ) diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 1425699455..37a9ed58bc 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -221,7 +221,9 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str): provider_uid = Provider.objects.get(id=provider_id).uid # Generate and ensure the output directory exists - output_directory = _generate_output_directory(DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id) + output_directory = _generate_output_directory( + DJANGO_TMP_OUTPUT_DIRECTORY, provider_uid, tenant_id, scan_id + ) # Define auxiliary variables output_writers = {} @@ -233,7 +235,9 @@ def generate_outputs(scan_id: str, provider_id: str, tenant_id: str): findings_qs = Finding.objects.filter(scan_id=scan_id).order_by("uid") # Process findings in batches - for batch, is_last_batch in batched(findings_qs.iterator(), DJANGO_FINDINGS_BATCH_SIZE): + for batch, is_last_batch in batched( + findings_qs.iterator(), DJANGO_FINDINGS_BATCH_SIZE + ): finding_outputs = [ FindingOutput.transform_api_finding(finding) for finding in batch ]