Compare commits

..
Author SHA1 Message Date
César Arroba 5be514a991 fix(api): fail migrations fast when blocked on a lock
Set lock_timeout on the migration session so a metadata-only migration
blocked waiting for ACCESS EXCLUSIVE on a busy table fails loudly instead
of queueing every later reader behind it. Configurable via
DJANGO_MIGRATION_LOCK_TIMEOUT (default 5s, 0 disables it).
2026-07-15 18:49:48 +02:00
Prowler Botandprowler-bot 0d899b3076 chore(release): Bump versions to v5.35.0 (#12004)
Co-authored-by: prowler-bot <179230569+prowler-bot@users.noreply.github.com>
2026-07-15 18:16:32 +02:00
48 changed files with 359 additions and 1149 deletions
+3 -3
View File
@@ -72,8 +72,8 @@ NEO4J_APOC_IMPORT_FILE_ENABLED=false
NEO4J_APOC_IMPORT_FILE_USE_NEO4J_CONFIG=true
NEO4J_APOC_TRIGGER_ENABLED=false
NEO4J_DBMS_CONNECTOR_BOLT_LISTEN_ADDRESS=0.0.0.0:7687
# Attack Paths graph settings
ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE=1000
# Neo4j Prowler settings
ATTACK_PATHS_BATCH_SIZE=1000
ATTACK_PATHS_SERVICE_UNAVAILABLE_MAX_RETRIES=3
ATTACK_PATHS_READ_QUERY_TIMEOUT_SECONDS=30
ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES=250
@@ -158,7 +158,7 @@ SENTRY_RELEASE=local
# REO_DEV_CLIENT_ID=
#### Prowler release version ####
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.34.1
NEXT_PUBLIC_PROWLER_RELEASE_VERSION=v5.35.0
# Social login credentials
SOCIAL_GOOGLE_OAUTH_CALLBACK_URL="${AUTH_URL}/api/auth/callback/google"
@@ -1 +0,0 @@
Attack Paths scans handle provider deletion races cleanly, detect stale tasks after 16 hours, use backend-specific graph synchronization batches, and report exhausted Neptune write retries with the original database error
@@ -1 +0,0 @@
`attack-paths-scan-perform` Celery tasks now use the configurable long-task time limits instead of the six-hour defaults
@@ -1 +0,0 @@
Jira integration credentials only accept bare Atlassian site names containing letters, numbers, and hyphens
@@ -0,0 +1 @@
Database migrations now give up after a few seconds when another query holds the table lock, instead of stalling every request that touches that table
@@ -1 +0,0 @@
Social account linking requires a verified matching email from both the identity provider and the existing user account without sending account connection notifications
+2 -2
View File
@@ -45,7 +45,7 @@ dependencies = [
"gunicorn==26.0.0",
"uvloop==0.22.1",
"lxml==6.1.0",
"prowler @ git+https://github.com/prowler-cloud/prowler.git@v5.34",
"prowler @ git+https://github.com/prowler-cloud/prowler.git@master",
"psycopg2-binary==2.9.9",
"pytest-celery[redis] (==1.3.0)",
"sentry-sdk[django] (==2.56.0)",
@@ -71,7 +71,7 @@ name = "prowler-api"
package-mode = false
# Needed for the SDK compatibility
requires-python = ">=3.11,<3.13"
version = "1.35.1"
version = "1.36.0"
# Shared ruff baseline (kept in sync with mcp_server/pyproject.toml).
# target-version tracks this project's lowest supported Python.
+2 -21
View File
@@ -1,5 +1,3 @@
from allauth.account.models import EmailAddress
from allauth.core.exceptions import ImmediateHttpResponse
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
from api.db_router import MainRouter
from api.db_utils import rls_transaction
@@ -13,7 +11,6 @@ from api.models import (
)
from api.utils import accept_invitation_for_user
from django.db import transaction
from django.http import HttpResponseForbidden
class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
@@ -41,13 +38,8 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
return None
def pre_social_login(self, request, sociallogin):
# The provider account is already bound, so no email-based linking is needed.
if sociallogin.account.pk:
return
# Prefer the normalized email populated by allauth. GitHub can return the
# primary email separately from the profile stored in extra_data.
email = sociallogin.user.email or sociallogin.account.extra_data.get("email")
# Link existing accounts with the same email address
email = sociallogin.account.extra_data.get("email")
if sociallogin.provider.id == "saml":
# For SAML, the asserted NameID email cannot be trusted on its own:
# any tenant can claim any email domain in its SAML configuration. To
@@ -88,17 +80,6 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter):
if email:
existing_user = self.get_user_by_email(email)
if existing_user:
email_is_verified = EmailAddress.objects.filter(
user=existing_user,
email__iexact=email,
verified=True,
).exists()
provider_verified_email = any(
address.verified and address.email.casefold() == email.casefold()
for address in sociallogin.email_addresses
)
if not email_is_verified or not provider_verified_email:
raise ImmediateHttpResponse(HttpResponseForbidden())
sociallogin.connect(request, existing_user)
def save_user(self, request, sociallogin, form=None):
@@ -27,7 +27,6 @@ from django.conf import (
MAX_CUSTOM_QUERY_NODES = env.int("ATTACK_PATHS_MAX_CUSTOM_QUERY_NODES", default=250)
TEMP_DB_PREFIX = "db-tmp-scan-"
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
# Exceptions
@@ -45,10 +44,6 @@ class GraphDatabaseQueryException(Exception):
return self.message
class NeptuneWriteRetryExhaustedException(GraphDatabaseQueryException):
pass
class WriteQueryNotAllowedException(GraphDatabaseQueryException):
pass
@@ -10,28 +10,6 @@ import neo4j.exceptions
logger = logging.getLogger(__name__)
class RetryExhaustedError(Exception):
def __init__(
self,
*,
retry_context: str,
method_name: str,
attempts: int,
elapsed_seconds: float,
last_error: Exception,
) -> None:
self.retry_context = retry_context
self.method_name = method_name
self.attempts = attempts
self.elapsed_seconds = elapsed_seconds
self.last_error = last_error
last_message = getattr(last_error, "message", None) or str(last_error)
super().__init__(
f"{retry_context} {method_name} failed after {attempts} attempts over "
f"{elapsed_seconds:.3f}s. Last error: {last_message}"
)
class RetryableSession:
"""Wrapper around ``neo4j.Session`` with a refreshable retry policy."""
@@ -41,13 +19,11 @@ class RetryableSession:
max_retries: int,
retry_if: Callable[[Exception], bool] | None = None,
initial_retry_delay_seconds: float = 0,
retry_context: str | None = None,
) -> None:
self._session_factory = session_factory
self._max_retries = max(0, max_retries)
self._retry_if = retry_if
self._initial_retry_delay_seconds = max(0.0, initial_retry_delay_seconds)
self._retry_context = retry_context
self._session = self._session_factory()
def close(self) -> None:
@@ -78,7 +54,6 @@ class RetryableSession:
def _call_with_retry(self, method_name: str, *args: Any, **kwargs: Any) -> Any:
attempt = 0
last_exc: Exception | None = None
started_at = time.monotonic()
while attempt <= self._max_retries:
try:
@@ -93,38 +68,17 @@ class RetryableSession:
attempt += 1
if attempt > self._max_retries:
if self._retry_context is not None:
raise RetryExhaustedError(
retry_context=self._retry_context,
method_name=method_name,
attempts=attempt,
elapsed_seconds=time.monotonic() - started_at,
last_error=exc,
) from exc
raise
delay = self._retry_delay(attempt)
if self._retry_context is not None:
error_message = getattr(exc, "message", None) or str(exc)
logger.warning(
"%s %s failed with %s: %s; retry %s/%s in %.3fs",
self._retry_context,
method_name,
type(exc).__name__,
error_message,
attempt,
self._max_retries,
delay,
)
else:
logger.warning(
"Graph session %s failed with %s; retry %s/%s in %.3fs",
method_name,
type(exc).__name__,
attempt,
self._max_retries,
delay,
)
logger.warning(
"Graph session %s failed with %s; retry %s/%s in %.3fs",
method_name,
type(exc).__name__,
attempt,
self._max_retries,
delay,
)
self._refresh_session()
if delay:
time.sleep(delay)
@@ -15,8 +15,6 @@ class SinkDatabase(Protocol):
has a single graph, and isolation is label-based).
"""
sync_batch_size: int
def init(self) -> None: ...
def close(self) -> None: ...
@@ -54,8 +54,6 @@ DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
class Neo4jSink(SinkDatabase):
"""Neo4j-backed sink. Multi-database cluster; tenant isolation is physical."""
sync_batch_size = env.int("ATTACK_PATHS_NEO4J_SYNC_BATCH_SIZE", default=1000)
def __init__(self) -> None:
self._driver: neo4j.Driver | None = None
self._lock = threading.Lock()
@@ -205,7 +203,7 @@ class Neo4jSink(SinkDatabase):
"""
from api.attack_paths.database import GraphDatabaseQueryException
from tasks.jobs.attack_paths.config import (
GRAPH_MUTATION_BATCH_SIZE,
BATCH_SIZE,
PROVIDER_RESOURCE_LABEL,
get_provider_label,
)
@@ -253,7 +251,7 @@ class Neo4jSink(SinkDatabase):
total_key="rels",
deleted_key="deleted_rels",
initial_total=deleted_relationships,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
batch_size=BATCH_SIZE,
drop_t0=drop_t0,
)
relationship_batches += phase_batches
@@ -272,7 +270,7 @@ class Neo4jSink(SinkDatabase):
total_key="nodes",
deleted_key="deleted_nodes",
initial_total=0,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
batch_size=BATCH_SIZE,
drop_t0=drop_t0,
)
@@ -25,7 +25,7 @@ from urllib.parse import urlsplit
import neo4j
import neo4j.exceptions
from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError
from api.attack_paths.retryable_session import RetryableSession
from api.attack_paths.sink.base import SinkDatabase
from api.attack_paths.sink.drop import (
NODE_DELETE_QUERY_TEMPLATE,
@@ -85,8 +85,6 @@ def _is_retryable_write_error(exc: Exception) -> bool:
class NeptuneSink(SinkDatabase):
"""Neptune-backed sink. Single database; isolation is label-based."""
sync_batch_size = env.int("ATTACK_PATHS_NEPTUNE_SYNC_BATCH_SIZE", default=500)
def __init__(self) -> None:
self._writer: neo4j.Driver | None = None
self._reader: neo4j.Driver | None = None
@@ -208,7 +206,6 @@ class NeptuneSink(SinkDatabase):
from api.attack_paths.database import (
ClientStatementException,
GraphDatabaseQueryException,
NeptuneWriteRetryExhaustedException,
WriteQueryNotAllowedException,
)
@@ -230,17 +227,9 @@ class NeptuneSink(SinkDatabase):
initial_retry_delay_seconds=(
NEPTUNE_WRITE_RETRY_DELAY_SECONDS if is_write_session else 0
),
retry_context="Neptune write" if is_write_session else None,
)
yield session_wrapper
except RetryExhaustedError as exc:
last_error = exc.last_error
raise NeptuneWriteRetryExhaustedException(
message=str(exc),
code=getattr(last_error, "code", None),
) from last_error
except neo4j.exceptions.Neo4jError as exc:
if (
default_access_mode == neo4j.READ_ACCESS
@@ -302,7 +291,7 @@ class NeptuneSink(SinkDatabase):
graph's branching factor.
"""
from tasks.jobs.attack_paths.config import (
GRAPH_MUTATION_BATCH_SIZE,
BATCH_SIZE,
PROVIDER_RESOURCE_LABEL,
get_provider_label,
)
@@ -341,7 +330,7 @@ class NeptuneSink(SinkDatabase):
total_key="rels",
deleted_key="deleted_rels",
initial_total=deleted_relationships,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
batch_size=BATCH_SIZE,
drop_t0=drop_t0,
)
relationship_batches += phase_batches
@@ -360,7 +349,7 @@ class NeptuneSink(SinkDatabase):
total_key="nodes",
deleted_key="deleted_nodes",
initial_total=0,
batch_size=GRAPH_MUTATION_BATCH_SIZE,
batch_size=BATCH_SIZE,
drop_t0=drop_t0,
)
+7 -22
View File
@@ -1,13 +1,12 @@
import uuid
from functools import wraps
from api.attack_paths.database import GraphDatabaseQueryException
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY, rls_transaction
from api.exceptions import ProviderDeletedException
from api.models import Membership, Provider, Scan, Tenant
from api.models import Provider, Scan
from django.core.exceptions import ObjectDoesNotExist
from django.db import DEFAULT_DB_ALIAS, DatabaseError, connection, transaction
from django.db import DatabaseError, connection, transaction
from rest_framework_json_api.serializers import ValidationError
@@ -76,11 +75,9 @@ def handle_provider_deletion(func):
"""
Decorator that raises `ProviderDeletedException` if provider was deleted during execution.
Catches `ObjectDoesNotExist`, `DatabaseError` (including `IntegrityError`), and
`GraphDatabaseQueryException`, checks if provider still exists, and raises
`ProviderDeletedException` if not. Graph database errors also check whether the
tenant still exists and has memberships. Otherwise, re-raises the original
exception.
Catches `ObjectDoesNotExist` and `DatabaseError` (including `IntegrityError`), checks if
provider still exists, and raises `ProviderDeletedException` if not. Otherwise,
re-raises original exception.
Requires `tenant_id` and `provider_id` in kwargs.
@@ -95,16 +92,11 @@ def handle_provider_deletion(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except (ObjectDoesNotExist, DatabaseError, GraphDatabaseQueryException) as exc:
except (ObjectDoesNotExist, DatabaseError):
tenant_id = kwargs.get("tenant_id")
provider_id = kwargs.get("provider_id")
database_alias = (
DEFAULT_DB_ALIAS
if isinstance(exc, GraphDatabaseQueryException)
else READ_REPLICA_ALIAS
)
with rls_transaction(tenant_id, using=database_alias):
with rls_transaction(tenant_id, using=READ_REPLICA_ALIAS):
if provider_id is None:
scan_id = kwargs.get("scan_id")
if scan_id is None:
@@ -121,13 +113,6 @@ def handle_provider_deletion(func):
raise ProviderDeletedException(
f"Provider '{provider_id}' was deleted during the scan"
) from None
if isinstance(exc, GraphDatabaseQueryException) and (
not Tenant.objects.filter(pk=tenant_id).exists()
or not Membership.objects.filter(tenant_id=tenant_id).exists()
):
raise ProviderDeletedException(
f"Tenant '{tenant_id}' was deleted during the scan"
) from None
raise
return wrapper
@@ -0,0 +1,28 @@
from config.env import env
from django.core.management.commands.migrate import Command as MigrateCommand
from django.db import connections
# Any value Postgres accepts for lock_timeout ("5s", "500ms", ...). "0" disables
# it, the escape hatch for a release whose DDL is expected to wait.
MIGRATION_LOCK_TIMEOUT = env.str("DJANGO_MIGRATION_LOCK_TIMEOUT", default="5s")
SET_LOCK_TIMEOUT_QUERY = "SELECT set_config('lock_timeout', %s, FALSE);"
class Command(MigrateCommand):
help = (
f"{MigrateCommand.help} Applies lock_timeout to the migration session so DDL "
"blocked on a lock fails instead of queueing every later reader behind it."
)
def handle(self, *args, **options):
connection = connections[options["database"]]
# is_local=FALSE keeps the setting alive across each migration's own
# transaction. Unlike the RLS tenant variable this is safe to leave on the
# session: migrate owns its connection for the life of the process and
# never returns it to a pool.
with connection.cursor() as cursor:
cursor.execute(SET_LOCK_TIMEOUT_QUERY, [MIGRATION_LOCK_TIMEOUT])
return super().handle(*args, **options)
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: 3.0.3
info:
title: Prowler API
version: 1.35.1
version: 1.36.0
description: |-
Prowler API specification.
+15 -171
View File
@@ -2,18 +2,11 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from allauth.account import app_settings as account_app_settings
from allauth.account.models import EmailAddress
from allauth.core import context
from allauth.core.exceptions import ImmediateHttpResponse
from allauth.socialaccount import app_settings as socialaccount_app_settings
from allauth.socialaccount.internal.flows.login import complete_login
from allauth.socialaccount.models import SocialAccount, SocialLogin
from allauth.socialaccount.models import SocialLogin
from api.adapters import ProwlerSocialAccountAdapter
from api.db_router import MainRouter
from api.models import Invitation, Membership, SAMLConfiguration, Tenant
from django.contrib.auth import get_user_model
from django.core import mail
User = get_user_model()
@@ -47,7 +40,6 @@ def _saml_request(rf, organization_slug):
def _saml_sociallogin(user):
sociallogin = MagicMock(spec=SocialLogin)
sociallogin.account = MagicMock()
sociallogin.account.pk = None
sociallogin.provider = MagicMock()
sociallogin.provider.id = "saml"
sociallogin.account.extra_data = {}
@@ -56,59 +48,6 @@ def _saml_sociallogin(user):
return sociallogin
def _oauth_sociallogin(
user,
*,
provider="google",
provider_email_verified=True,
include_extra_email=True,
):
sociallogin = MagicMock(spec=SocialLogin)
sociallogin.account = MagicMock()
sociallogin.account.pk = None
sociallogin.provider = MagicMock()
sociallogin.provider.id = provider
sociallogin.account.extra_data = (
{"email": user.email} if include_extra_email else {}
)
sociallogin.email_addresses = [
EmailAddress(
email=user.email,
verified=provider_email_verified,
primary=True,
)
]
sociallogin.user = user
sociallogin.connect = MagicMock()
return sociallogin
def _real_oauth_sociallogin(user, uid):
provider = MagicMock()
provider.id = "google"
provider.app = None
provider.get_settings.return_value = {}
return SocialLogin(
user=user,
account=SocialAccount(
provider="google",
uid=uid,
extra_data={"email": user.email},
),
email_addresses=[EmailAddress(email=user.email, verified=True, primary=True)],
provider=provider,
)
def _verify_local_email(user):
return EmailAddress.objects.create(
user=user,
email=user.email,
verified=True,
primary=True,
)
@pytest.mark.django_db
class TestProwlerSocialAccountAdapter:
def test_get_user_by_email_returns_user(self, create_test_user):
@@ -218,7 +157,6 @@ class TestProwlerSocialAccountAdapter:
sociallogin = MagicMock(spec=SocialLogin)
sociallogin.account = MagicMock()
sociallogin.account.pk = None
sociallogin.provider = MagicMock()
sociallogin.user = MagicMock()
sociallogin.user.email = ""
@@ -230,119 +168,25 @@ class TestProwlerSocialAccountAdapter:
sociallogin.connect.assert_not_called()
def test_pre_social_login_blocks_unverified_local_email(self, create_test_user, rf):
"""A verified OAuth email must not claim an unverified local account."""
def test_pre_social_login_non_saml_links_by_email(self, create_test_user, rf):
"""Non-SAML providers (e.g. Google/GitHub) still link to an existing
local account by email; the tenant binding only applies to SAML."""
adapter = ProwlerSocialAccountAdapter()
sociallogin = _oauth_sociallogin(create_test_user)
with pytest.raises(ImmediateHttpResponse) as exc_info:
adapter.pre_social_login(rf.get("/"), sociallogin)
assert exc_info.value.response.status_code == 403
sociallogin.connect.assert_not_called()
def test_complete_oauth_login_does_not_link_unverified_local_email(
self, create_test_user, rf
):
"""Regression test for the complete pre-hijack account-linking flow."""
incoming_user = User(email=create_test_user.email)
incoming_user.set_unusable_password()
sociallogin = _real_oauth_sociallogin(
incoming_user,
uid="victim-google-account",
)
request = rf.get("/")
request.session = {}
with pytest.raises(ImmediateHttpResponse) as exc_info:
complete_login(request, sociallogin, raises=True)
assert exc_info.value.response.status_code == 403
assert not SocialAccount.objects.filter(
provider="google", uid="victim-google-account"
).exists()
def test_pre_social_login_allows_already_connected_account(
self, create_test_user, rf
):
"""Existing provider bindings do not need to relink on every login."""
adapter = ProwlerSocialAccountAdapter()
sociallogin = _oauth_sociallogin(create_test_user)
sociallogin.account.pk = "existing-social-account"
sociallogin = MagicMock(spec=SocialLogin)
sociallogin.account = MagicMock()
sociallogin.provider = MagicMock()
sociallogin.provider.id = "google"
sociallogin.account.extra_data = {"email": create_test_user.email}
sociallogin.user = create_test_user
sociallogin.connect = MagicMock()
adapter.pre_social_login(rf.get("/"), sociallogin)
sociallogin.connect.assert_not_called()
def test_pre_social_login_blocks_unverified_provider_email(
self, create_test_user, rf
):
"""An OAuth provider must prove ownership of the matching email."""
_verify_local_email(create_test_user)
adapter = ProwlerSocialAccountAdapter()
sociallogin = _oauth_sociallogin(
create_test_user,
provider="github",
provider_email_verified=False,
)
with pytest.raises(ImmediateHttpResponse) as exc_info:
adapter.pre_social_login(rf.get("/"), sociallogin)
assert exc_info.value.response.status_code == 403
sociallogin.connect.assert_not_called()
def test_pre_social_login_links_verified_emails(self, create_test_user, rf):
_verify_local_email(create_test_user)
adapter = ProwlerSocialAccountAdapter()
sociallogin = _oauth_sociallogin(create_test_user)
request = rf.get("/")
adapter.pre_social_login(request, sociallogin)
sociallogin.connect.assert_called_once_with(request, create_test_user)
def test_verified_social_account_link_does_not_send_notification(
self, create_test_user, rf
):
_verify_local_email(create_test_user)
sociallogin = _real_oauth_sociallogin(
create_test_user,
uid="verified-google-account",
)
request = rf.get("/")
with context.request_context(request):
ProwlerSocialAccountAdapter().pre_social_login(request, sociallogin)
assert SocialAccount.objects.filter(
provider="google",
uid="verified-google-account",
user=create_test_user,
).exists()
assert mail.outbox == []
def test_pre_social_login_uses_verified_email_missing_from_extra_data(
self, create_test_user, rf
):
"""GitHub can return its verified primary email outside extra_data."""
_verify_local_email(create_test_user)
adapter = ProwlerSocialAccountAdapter()
sociallogin = _oauth_sociallogin(
create_test_user,
provider="github",
include_extra_email=False,
)
request = rf.get("/")
adapter.pre_social_login(request, sociallogin)
sociallogin.connect.assert_called_once_with(request, create_test_user)
def test_social_account_linking_settings_are_fail_closed(self):
assert not socialaccount_app_settings.EMAIL_AUTHENTICATION
assert not socialaccount_app_settings.EMAIL_AUTHENTICATION_AUTO_CONNECT
assert not account_app_settings.EMAIL_NOTIFICATIONS
call_args = sociallogin.connect.call_args
assert call_args is not None
_, called_user = call_args[0]
assert called_user.email == create_test_user.email
def test_save_user_social_with_invitation_joins_invited_tenant(
self, rf, create_test_user, tenants_fixture
+1 -102
View File
@@ -2,12 +2,11 @@ import uuid
from unittest.mock import call, patch
import pytest
from api.attack_paths.database import GraphDatabaseQueryException
from api.db_utils import POSTGRES_TENANT_VAR, SET_CONFIG_QUERY
from api.decorators import handle_provider_deletion, set_tenant
from api.exceptions import ProviderDeletedException
from django.core.exceptions import ObjectDoesNotExist
from django.db import DEFAULT_DB_ALIAS, DatabaseError, IntegrityError
from django.db import DatabaseError, IntegrityError
@pytest.mark.django_db
@@ -205,106 +204,6 @@ class TestHandleProviderDeletionDecorator:
with pytest.raises(DatabaseError):
task_func(tenant_id=str(tenant.id), provider_id=str(provider.id))
@patch("api.decorators.rls_transaction")
@patch("api.decorators.Provider.objects.filter")
def test_graph_database_error_provider_missing_or_soft_deleted(
self, mock_provider_filter, mock_rls, tenants_fixture
):
tenant = tenants_fixture[0]
provider_id = str(uuid.uuid4())
mock_rls.return_value.__enter__ = lambda s: None
mock_rls.return_value.__exit__ = lambda s, *args: None
mock_provider_filter.return_value.exists.return_value = False
@handle_provider_deletion
def task_func(**kwargs):
raise GraphDatabaseQueryException("Temporary database not found")
with pytest.raises(ProviderDeletedException):
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
@patch("api.decorators.rls_transaction")
@patch("api.decorators.Tenant.objects.filter")
@patch("api.decorators.Provider.objects.filter")
def test_graph_database_error_tenant_missing(
self, mock_provider_filter, mock_tenant_filter, mock_rls, tenants_fixture
):
tenant = tenants_fixture[0]
provider_id = str(uuid.uuid4())
mock_rls.return_value.__enter__ = lambda s: None
mock_rls.return_value.__exit__ = lambda s, *args: None
mock_provider_filter.return_value.exists.return_value = True
mock_tenant_filter.return_value.exists.return_value = False
@handle_provider_deletion
def task_func(**kwargs):
raise GraphDatabaseQueryException("Temporary database not found")
with pytest.raises(ProviderDeletedException):
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
@patch("api.decorators.rls_transaction")
@patch("api.decorators.Membership.objects.filter")
@patch("api.decorators.Tenant.objects.filter")
@patch("api.decorators.Provider.objects.filter")
def test_graph_database_error_tenant_without_memberships(
self,
mock_provider_filter,
mock_tenant_filter,
mock_membership_filter,
mock_rls,
tenants_fixture,
):
tenant = tenants_fixture[0]
provider_id = str(uuid.uuid4())
mock_rls.return_value.__enter__ = lambda s: None
mock_rls.return_value.__exit__ = lambda s, *args: None
mock_provider_filter.return_value.exists.return_value = True
mock_tenant_filter.return_value.exists.return_value = True
mock_membership_filter.return_value.exists.return_value = False
@handle_provider_deletion
def task_func(**kwargs):
raise GraphDatabaseQueryException("Temporary database not found")
with pytest.raises(ProviderDeletedException):
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
@patch("api.decorators.rls_transaction")
@patch("api.decorators.Membership.objects.filter")
@patch("api.decorators.Tenant.objects.filter")
@patch("api.decorators.Provider.objects.filter")
def test_graph_database_error_active_provider_and_tenant_reraises(
self,
mock_provider_filter,
mock_tenant_filter,
mock_membership_filter,
mock_rls,
tenants_fixture,
):
tenant = tenants_fixture[0]
provider_id = str(uuid.uuid4())
graph_error = GraphDatabaseQueryException("Temporary database not found")
mock_rls.return_value.__enter__ = lambda s: None
mock_rls.return_value.__exit__ = lambda s, *args: None
mock_provider_filter.return_value.exists.return_value = True
mock_tenant_filter.return_value.exists.return_value = True
mock_membership_filter.return_value.exists.return_value = True
@handle_provider_deletion
def task_func(**kwargs):
raise graph_error
with pytest.raises(GraphDatabaseQueryException) as exc_info:
task_func(tenant_id=str(tenant.id), provider_id=provider_id)
assert exc_info.value is graph_error
mock_rls.assert_called_once_with(str(tenant.id), using=DEFAULT_DB_ALIAS)
def test_missing_provider_and_scan_raises_assertion(self, tenants_fixture):
"""Raises AssertionError when neither provider_id nor scan_id in kwargs."""
@@ -0,0 +1,179 @@
from unittest.mock import Mock, patch
import psycopg2
import pytest
from django.core.management import call_command
from django.core.management.commands.migrate import Command as DjangoMigrateCommand
from django.db import DEFAULT_DB_ALIAS, OperationalError, connections
from django.db.migrations import Migration
from django.db.migrations.operations.special import RunSQL
from django.db.migrations.state import ProjectState
def _show_lock_timeout(alias: str = DEFAULT_DB_ALIAS) -> str:
with connections[alias].cursor() as cursor:
cursor.execute("SHOW lock_timeout;")
return cursor.fetchone()[0]
@pytest.mark.django_db
class TestMigrateLockTimeout:
def test_api_migrate_command_shadows_django_builtin(self):
from django.core.management import get_commands
assert get_commands()["migrate"] == "api"
def test_lock_timeout_is_set_before_django_migrates(self):
observed = {}
def fake_handle(_self, *args, **options):
observed["lock_timeout"] = _show_lock_timeout()
with patch.object(DjangoMigrateCommand, "handle", fake_handle):
call_command("migrate")
assert observed["lock_timeout"] == "5s"
def test_lock_timeout_value_is_configurable(self):
observed = {}
def fake_handle(_self, *args, **options):
observed["lock_timeout"] = _show_lock_timeout()
with (
patch(
"api.management.commands.migrate.MIGRATION_LOCK_TIMEOUT",
"250ms",
),
patch.object(DjangoMigrateCommand, "handle", fake_handle),
):
call_command("migrate")
assert observed["lock_timeout"] == "250ms"
def test_lock_timeout_zero_disables_the_timeout(self):
observed = {}
def fake_handle(_self, *args, **options):
observed["lock_timeout"] = _show_lock_timeout()
with (
patch("api.management.commands.migrate.MIGRATION_LOCK_TIMEOUT", "0"),
patch.object(DjangoMigrateCommand, "handle", fake_handle),
):
call_command("migrate")
assert observed["lock_timeout"] == "0"
def test_lock_timeout_applies_to_the_requested_database(self):
with patch.object(DjangoMigrateCommand, "handle", Mock(return_value=None)):
call_command("migrate", database=DEFAULT_DB_ALIAS)
assert _show_lock_timeout() == "5s"
@pytest.mark.django_db(transaction=True)
class TestMigrateLockTimeoutFailureMode:
"""
Exercises the real rollback semantics against Postgres: a migration blocked on
a lock must fail fast and leave nothing half-applied.
"""
table = "lock_timeout_probe"
@pytest.fixture
def blocked_table(self, settings):
with connections[DEFAULT_DB_ALIAS].cursor() as cursor:
cursor.execute(f"DROP TABLE IF EXISTS {self.table};")
cursor.execute(f"CREATE TABLE {self.table} (id integer);")
db = settings.DATABASES[DEFAULT_DB_ALIAS]
blocker = psycopg2.connect(
dbname=db["NAME"],
user=db["USER"],
password=db["PASSWORD"],
host=db["HOST"],
port=db["PORT"],
)
with blocker.cursor() as cursor:
cursor.execute(f"LOCK TABLE {self.table} IN ACCESS EXCLUSIVE MODE;")
yield
blocker.rollback()
blocker.close()
with connections[DEFAULT_DB_ALIAS].cursor() as cursor:
cursor.execute(f"DROP TABLE IF EXISTS {self.table};")
def _column_exists(self, column: str) -> bool:
with connections[DEFAULT_DB_ALIAS].cursor() as cursor:
cursor.execute(
"SELECT 1 FROM information_schema.columns "
"WHERE table_name = %s AND column_name = %s;",
[self.table, column],
)
return cursor.fetchone() is not None
def _apply(self, atomic: bool):
connection = connections[DEFAULT_DB_ALIAS]
with connection.cursor() as cursor:
cursor.execute("SELECT set_config('lock_timeout', '250ms', FALSE);")
migration = type(
"ProbeMigration",
(Migration,),
{
"atomic": atomic,
"operations": [
RunSQL(f"CREATE TABLE {self.table}_first (id integer);"),
RunSQL(f"ALTER TABLE {self.table} ADD COLUMN blocked integer;"),
],
},
)("probe", "api")
try:
with connection.schema_editor(atomic=migration.atomic) as schema_editor:
migration.apply(ProjectState(), schema_editor, collect_sql=False)
finally:
with connection.cursor() as cursor:
cursor.execute("SELECT set_config('lock_timeout', '0', FALSE);")
def _first_table_exists(self) -> bool:
with connections[DEFAULT_DB_ALIAS].cursor() as cursor:
cursor.execute("SELECT to_regclass(%s);", [f"{self.table}_first"])
return cursor.fetchone()[0] is not None
def _drop_first_table(self):
with connections[DEFAULT_DB_ALIAS].cursor() as cursor:
cursor.execute(f"DROP TABLE IF EXISTS {self.table}_first;")
def test_atomic_migration_rolls_back_entirely(self, blocked_table):
# Fixture holds the lock the migration blocks on; pytest injects it by
# parameter name, so we reference it explicitly to keep static
# analysers from flagging it as unused.
del blocked_table
try:
with pytest.raises(OperationalError, match="lock timeout"):
self._apply(atomic=True)
assert not self._column_exists("blocked")
# The whole migration is one transaction, so the operation that ran
# before the blocked one is rolled back too: nothing half-applied.
assert not self._first_table_exists()
finally:
self._drop_first_table()
def test_non_atomic_migration_keeps_earlier_operations(self, blocked_table):
del blocked_table
try:
with pytest.raises(OperationalError, match="lock timeout"):
self._apply(atomic=False)
assert not self._column_exists("blocked")
# atomic = False has no surrounding transaction, so earlier operations
# survive while the migration stays unrecorded. Re-running it replays
# them. This is inherent to atomic = False, not to lock_timeout, but
# lock_timeout makes it reachable more often.
assert self._first_table_exists()
finally:
self._drop_first_table()
@@ -1,7 +1,7 @@
from unittest.mock import MagicMock, patch
import pytest
from api.attack_paths.retryable_session import RetryableSession, RetryExhaustedError
from api.attack_paths.retryable_session import RetryableSession
from neo4j.exceptions import ServiceUnavailable
@@ -24,7 +24,6 @@ class TestRetryableSession:
max_retries=3,
retry_if=lambda exc: exc is retryable_error,
initial_retry_delay_seconds=2,
retry_context="Neptune write",
)
assert session.execute_write(work) == "success"
@@ -55,7 +54,6 @@ class TestRetryableSession:
max_retries=3,
retry_if=lambda _: False,
initial_retry_delay_seconds=2,
retry_context="Neptune write",
)
with pytest.raises(RuntimeError) as exc_info:
@@ -85,81 +83,3 @@ class TestRetryableSession:
driver_sessions[0].close.assert_called_once_with()
driver_sessions[1].close.assert_called_once_with()
driver_sessions[2].close.assert_not_called()
def test_retry_exhaustion_with_context_reports_attempts_and_elapsed_time(self):
error = RuntimeError("still retryable")
driver_sessions = [MagicMock() for _ in range(3)]
for driver_session in driver_sessions:
driver_session.execute_write.side_effect = error
session = RetryableSession(
session_factory=MagicMock(side_effect=driver_sessions),
max_retries=2,
retry_if=lambda _: True,
retry_context="Neptune write",
)
with (
patch(
"api.attack_paths.retryable_session.time.monotonic",
side_effect=[100.0, 127.1234],
),
pytest.raises(RetryExhaustedError) as exc_info,
):
session.execute_write(MagicMock())
assert exc_info.value.method_name == "execute_write"
assert exc_info.value.attempts == 3
assert exc_info.value.elapsed_seconds == pytest.approx(27.1234)
assert exc_info.value.last_error is error
assert exc_info.value.__cause__ is error
assert str(exc_info.value) == (
"Neptune write execute_write failed after 3 attempts over 27.123s. "
"Last error: still retryable"
)
def test_retry_exhaustion_with_zero_retries_reports_one_attempt(self):
error = ServiceUnavailable("still unavailable")
driver_session = MagicMock()
driver_session.execute_write.side_effect = error
session = RetryableSession(
session_factory=MagicMock(return_value=driver_session),
max_retries=0,
retry_context="Neptune write",
)
with pytest.raises(RetryExhaustedError) as exc_info:
session.execute_write(MagicMock())
assert exc_info.value.attempts == 1
@patch("api.attack_paths.retryable_session.time.sleep")
@patch("api.attack_paths.retryable_session.random.uniform", return_value=3.0)
def test_contextual_retry_warning_includes_original_error(
self, _mock_uniform, _mock_sleep
):
error = RuntimeError("retryable detail")
first_session = MagicMock()
first_session.execute_write.side_effect = error
second_session = MagicMock()
second_session.execute_write.return_value = "success"
session = RetryableSession(
session_factory=MagicMock(side_effect=[first_session, second_session]),
max_retries=1,
retry_if=lambda _: True,
initial_retry_delay_seconds=2,
retry_context="Neptune write",
)
with patch("api.attack_paths.retryable_session.logger.warning") as mock_warning:
assert session.execute_write(MagicMock()) == "success"
mock_warning.assert_called_once_with(
"%s %s failed with %s: %s; retry %s/%s in %.3fs",
"Neptune write",
"execute_write",
"RuntimeError",
"retryable detail",
1,
1,
3.0,
)
-40
View File
@@ -1,7 +1,6 @@
import logging
from unittest.mock import MagicMock, patch
import pytest
from config.settings import sentry as sentry_settings
from config.settings.sentry import before_send
@@ -83,45 +82,6 @@ def test_before_send_passes_through_non_ignored_log():
assert result == event
def test_before_send_ignores_cartography_missing_temporary_database_log():
log_record = _make_log_record(
msg="Cartography job failed with %s for database %s",
name="cartography.graph.job",
args=(
"Neo.ClientError.Database.DatabaseNotFound",
"db-tmp-scan-12345678",
),
)
event = MagicMock()
assert before_send(event, {"log_record": log_record}) is None
@pytest.mark.parametrize(
("logger_name", "message"),
[
(
"cartography.graph.job.worker",
"Neo.ClientError.Database.DatabaseNotFound for db-tmp-scan-12345678",
),
(
"cartography.graph.job",
"DatabaseNotFound for db-tmp-scan-12345678",
),
(
"cartography.graph.job",
"Neo.ClientError.Database.DatabaseNotFound for db-tenant-12345678",
),
],
)
def test_before_send_passes_through_similar_cartography_logs(logger_name, message):
log_record = _make_log_record(msg=message, name=logger_name)
event = MagicMock()
assert before_send(event, {"log_record": log_record}) is event
def test_before_send_passes_through_non_ignored_exception():
"""Test that before_send passes through exceptions that don't contain ignored exceptions."""
exc_info = (Exception, Exception("Some other error message"), None)
+1 -57
View File
@@ -1,8 +1,5 @@
import pytest
from api.v1.serializer_utils.integrations import (
JiraCredentialSerializer,
S3ConfigSerializer,
)
from api.v1.serializer_utils.integrations import S3ConfigSerializer
from api.v1.serializers import ImageProviderSecret, KubernetesProviderSecret
from rest_framework.exceptions import ValidationError
@@ -103,59 +100,6 @@ class TestS3ConfigSerializer:
assert "output_directory" in serializer.errors
class TestJiraCredentialSerializer:
@pytest.mark.parametrize(
"domain",
(
"a",
"prowler",
"prowler-domain",
"A1-b2-C3",
"a" * 63,
),
)
def test_valid_site_name(self, domain):
serializer = JiraCredentialSerializer(
data={
"user_mail": "testing@prowler.com",
"api_token": "fake-api-token",
"domain": domain,
}
)
assert serializer.is_valid(), serializer.errors
@pytest.mark.parametrize(
"domain",
(
"169.254.169.254#",
"internal/service",
"internal?target",
"internal\\target",
"internal:8000",
"user@internal",
"example.atlassian.net",
"-prowler",
"prowler-",
"a" * 64,
" prowler",
"prowler ",
"prowler\n",
),
)
def test_invalid_site_name(self, domain):
serializer = JiraCredentialSerializer(
data={
"user_mail": "testing@prowler.com",
"api_token": "fake-api-token",
"domain": domain,
}
)
assert not serializer.is_valid()
assert "domain" in serializer.errors
class TestImageProviderSecret:
"""Test cases for ImageProviderSecret validation."""
+1 -56
View File
@@ -11,11 +11,7 @@ from unittest.mock import MagicMock, patch
import neo4j
import pytest
from api.attack_paths import sink as sink_module
from api.attack_paths.database import (
GraphDatabaseQueryException,
NeptuneWriteRetryExhaustedException,
)
from api.attack_paths.retryable_session import RetryExhaustedError
from api.attack_paths.database import GraphDatabaseQueryException
from api.attack_paths.sink import factory
from api.attack_paths.sink.neo4j import DATABASE_NOT_FOUND_CODE, Neo4jSink
from api.attack_paths.sink.neptune import (
@@ -127,14 +123,6 @@ class TestSinkFactory:
assert mock_driver.call_count == 1
def test_neo4j_sync_batch_size_defaults_to_1000():
assert Neo4jSink.sync_batch_size == 1000
def test_neptune_sync_batch_size_defaults_to_500():
assert NeptuneSink.sync_batch_size == 500
class TestGetBackendForScan:
"""``get_backend_for_scan`` routes by the row's recorded sink backend."""
@@ -384,7 +372,6 @@ class TestNeptuneRetryPolicy:
assert (
kwargs["initial_retry_delay_seconds"] == NEPTUNE_WRITE_RETRY_DELAY_SECONDS
)
assert kwargs["retry_context"] == "Neptune write"
@patch("api.attack_paths.sink.neptune.RetryableSession")
def test_reader_session_does_not_enable_write_retry_policy(self, retryable_session):
@@ -397,48 +384,6 @@ class TestNeptuneRetryPolicy:
kwargs = retryable_session.call_args.kwargs
assert kwargs["retry_if"] is None
assert kwargs["initial_retry_delay_seconds"] == 0
assert kwargs["retry_context"] is None
def test_writer_retry_exhaustion_preserves_neptune_error_details(self):
message = (
"Unexpected server exception 'Operation failed due to conflicting "
"concurrent operations (please retry), 0 transactions are currently "
"rolling back.'"
)
error = neo4j.exceptions.Neo4jError._hydrate_neo4j(
code="BoltProtocol.unexpectedException",
message=message,
)
retry_error = RetryExhaustedError(
retry_context="Neptune write",
method_name="execute_write",
attempts=4,
elapsed_seconds=27.1234,
last_error=error,
)
sink = NeptuneSink()
driver = MagicMock()
retryable_session = MagicMock()
retryable_session.execute_write.side_effect = retry_error
with (
patch.object(sink, "_get_writer", return_value=driver),
patch(
"api.attack_paths.sink.neptune.RetryableSession",
return_value=retryable_session,
),
pytest.raises(NeptuneWriteRetryExhaustedException) as exc_info,
):
with sink.get_session() as session:
session.execute_write(MagicMock())
assert exc_info.value.code == "BoltProtocol.unexpectedException"
assert str(exc_info.value) == (
"BoltProtocol.unexpectedException: Neptune write execute_write failed "
"after 4 attempts over 27.123s. Last error: "
f"{message}"
)
assert exc_info.value.__cause__ is error
class TestNeptuneSinkDropSubgraph:
+5 -5
View File
@@ -856,7 +856,7 @@ class TestProwlerIntegrationConnectionTest:
integration.credentials = {
"user_mail": "test@example.com",
"api_token": "test_api_token",
"domain": "example",
"domain": "example.atlassian.net",
}
integration.configuration = {}
@@ -884,7 +884,7 @@ class TestProwlerIntegrationConnectionTest:
mock_jira_class.test_connection.assert_called_once_with(
user_mail="test@example.com",
api_token="test_api_token",
domain="example",
domain="example.atlassian.net",
raise_on_exception=False,
)
@@ -917,7 +917,7 @@ class TestProwlerIntegrationConnectionTest:
integration.credentials = {
"user_mail": "invalid@example.com",
"api_token": "invalid_token",
"domain": "invalid",
"domain": "invalid.atlassian.net",
}
integration.configuration = {}
@@ -942,7 +942,7 @@ class TestProwlerIntegrationConnectionTest:
mock_jira_class.test_connection.assert_called_once_with(
user_mail="invalid@example.com",
api_token="invalid_token",
domain="invalid",
domain="invalid.atlassian.net",
raise_on_exception=False,
)
@@ -970,7 +970,7 @@ class TestProwlerIntegrationConnectionTest:
integration.credentials = {
"user_mail": "test@example.com",
"api_token": "test_api_token",
"domain": "example",
"domain": "example.atlassian.net",
}
integration.configuration = {
"issue_types": {"OLD_PROJ": ["Task"]}, # Existing configuration
-88
View File
@@ -13423,45 +13423,6 @@ class TestIntegrationViewSet:
)
assert "credentials" not in response.json()["data"]["attributes"]
@pytest.mark.parametrize(
"domain",
(
"169.254.169.254#",
"internal/service",
"internal?target",
"internal\\target",
"internal:8000",
"user@internal",
),
)
def test_integrations_create_jira_rejects_invalid_domain(
self, authenticated_client, domain
):
data = {
"data": {
"type": "integrations",
"attributes": {
"integration_type": Integration.IntegrationChoices.JIRA,
"configuration": {},
"credentials": {
"domain": domain,
"api_token": "fake-api-token",
"user_mail": "testing@prowler.com",
},
"enabled": True,
},
}
}
response = authenticated_client.post(
reverse("integration-list"),
data=json.dumps(data),
content_type="application/vnd.api+json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert Integration.objects.count() == 0
def test_integrations_create_valid_relationships(
self,
authenticated_client,
@@ -14002,55 +13963,6 @@ class TestIntegrationViewSet:
assert "projects" in configuration
assert "issue_types" in configuration
def test_integrations_update_jira_rejects_invalid_domain(
self, authenticated_client
):
create_data = {
"data": {
"type": "integrations",
"attributes": {
"integration_type": Integration.IntegrationChoices.JIRA,
"configuration": {},
"credentials": {
"user_mail": "test@example.com",
"api_token": "fake-api-token",
"domain": "original-domain",
},
"enabled": True,
},
}
}
create_response = authenticated_client.post(
reverse("integration-list"),
data=json.dumps(create_data),
content_type="application/vnd.api+json",
)
assert create_response.status_code == status.HTTP_201_CREATED
integration_id = create_response.json()["data"]["id"]
update_data = {
"data": {
"type": "integrations",
"id": integration_id,
"attributes": {
"credentials": {
"user_mail": "test@example.com",
"api_token": "fake-api-token",
"domain": "169.254.169.254#",
}
},
}
}
response = authenticated_client.patch(
reverse("integration-detail", kwargs={"pk": integration_id}),
data=json.dumps(update_data),
content_type="application/vnd.api+json",
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
integration = Integration.objects.get(id=integration_id)
assert integration.credentials["domain"] == "original-domain"
@pytest.mark.django_db
class TestSAMLTokenValidation:
@@ -5,10 +5,6 @@ from api.v1.serializer_utils.base import BaseValidateSerializer
from drf_spectacular.utils import extend_schema_field
from rest_framework_json_api import serializers
ATLASSIAN_SITE_NAME_REGEX = re.compile(
r"\A[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\Z"
)
class S3ConfigSerializer(BaseValidateSerializer):
bucket_name = serializers.CharField()
@@ -101,17 +97,7 @@ class AWSCredentialSerializer(BaseValidateSerializer):
class JiraCredentialSerializer(BaseValidateSerializer):
user_mail = serializers.EmailField(required=True)
api_token = serializers.CharField(required=True)
domain = serializers.RegexField(
regex=ATLASSIAN_SITE_NAME_REGEX,
required=True,
trim_whitespace=False,
error_messages={
"invalid": (
"Domain must be a valid Atlassian site name containing only "
"letters, numbers, and hyphens."
)
},
)
domain = serializers.CharField(required=True)
class Meta:
resource_name = "integrations"
@@ -184,10 +170,7 @@ class JiraCredentialSerializer(BaseValidateSerializer):
},
"domain": {
"type": "string",
"description": "The Jira site name without the '.atlassian.net' suffix (e.g., 'your-domain').",
"minLength": 1,
"maxLength": 63,
"pattern": "^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$",
"description": "The JIRA domain/instance URL (e.g., 'your-domain.atlassian.net').",
},
},
"required": ["user_mail", "api_token", "domain"],
-1
View File
@@ -74,7 +74,6 @@ celery_app.conf.task_annotations = {
for name in (
"scan-perform",
"scan-perform-scheduled",
"attack-paths-scan-perform",
"provider-deletion",
"tenant-deletion",
)
+2 -2
View File
@@ -312,8 +312,8 @@ ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES = env.int(
"ATTACK_PATHS_SCAN_INACTIVITY_THRESHOLD_MINUTES", 30
)
ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES = env.int(
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 960
) # 16h
"ATTACK_PATHS_SCAN_STALE_THRESHOLD_MINUTES", 2880
) # 48h
# Selects where the persistent attack-paths graph is stored. The scan
# temporary database is always Neo4j; only the sink is configurable.
@@ -91,13 +91,6 @@ def before_send(event, hint):
log_msg = log_record.getMessage()
log_lvl = log_record.levelno
if (
getattr(log_record, "name", "") == "cartography.graph.job"
and "Neo.ClientError.Database.DatabaseNotFound" in log_msg
and "db-tmp-scan-" in log_msg
):
return None
# The Neo4j driver logs transient connection errors (defunct
# connections, resets) at ERROR level via the `neo4j.io` logger.
# `RetryableSession` handles these with retries. If all retries
@@ -13,17 +13,16 @@ GITHUB_OAUTH_CALLBACK_URL = env("SOCIAL_GITHUB_OAUTH_CALLBACK_URL", default="")
ACCOUNT_LOGIN_METHODS = {"email"} # Use Email / Password authentication
ACCOUNT_SIGNUP_FIELDS = ["email*", "password1*", "password2*"]
ACCOUNT_EMAIL_VERIFICATION = "none" # Do not require email confirmation
ACCOUNT_EMAIL_NOTIFICATIONS = False
ACCOUNT_USER_MODEL_USERNAME_FIELD = None
REST_AUTH = {
"TOKEN_MODEL": None,
"REST_USE_JWT": True,
}
# django-allauth (social)
# Email-based account matching is handled by ProwlerSocialAccountAdapter, which
# verifies both the provider email and the existing account email before linking.
SOCIALACCOUNT_EMAIL_AUTHENTICATION = False
SOCIALACCOUNT_EMAIL_AUTHENTICATION_AUTO_CONNECT = False
# Authenticate if local account with this email address already exists
SOCIALACCOUNT_EMAIL_AUTHENTICATION = True
# Connect local account and social account if local account with that email address already exists
SOCIALACCOUNT_EMAIL_AUTHENTICATION_AUTO_CONNECT = True
SOCIALACCOUNT_ADAPTER = "api.adapters.ProwlerSocialAccountAdapter"
@@ -8,8 +8,6 @@ import aioboto3
import boto3
import botocore
import neo4j
import neo4j.exceptions
from api.attack_paths.database import DATABASE_NOT_FOUND_CODE
from api.models import (
AttackPathsScan as ProwlerAPIAttackPathsScan,
)
@@ -349,12 +347,6 @@ def sync_aws_account(
)
except Exception as e:
if (
isinstance(e, neo4j.exceptions.Neo4jError)
and e.code == DATABASE_NOT_FOUND_CODE
):
raise
logger.info(
f"Synced function {func_name} for AWS account {prowler_api_provider.uid} in {time.perf_counter() - func_t0:.3f}s (FAILED)"
)
@@ -10,10 +10,13 @@ NormalizedList = _provider_config.NormalizedList
PROVIDER_CONFIGS = _provider_config.PROVIDER_CONFIGS
ProviderConfig = _provider_config.ProviderConfig
# Batch size for graph mutation operations (resource labeling and subgraph deletion)
GRAPH_MUTATION_BATCH_SIZE = env.int("ATTACK_PATHS_GRAPH_MUTATION_BATCH_SIZE", 1000)
# Batch size for Neo4j write operations (resource labeling, cleanup)
BATCH_SIZE = env.int("ATTACK_PATHS_BATCH_SIZE", 1000)
# Batch size for Postgres findings fetch (keyset pagination page size)
FINDINGS_BATCH_SIZE = env.int("ATTACK_PATHS_FINDINGS_BATCH_SIZE", 1000)
# Batch size for temp-to-tenant graph sync (nodes and relationships per cursor page)
SYNC_BATCH_SIZE = env.int("ATTACK_PATHS_SYNC_BATCH_SIZE", 1000)
# Neo4j internal labels (Prowler-specific, not provider-specific)
# - `Internet`: Singleton node representing external internet access for exposed-resource queries
# - `ProwlerFinding`: Label for finding nodes created by Prowler and linked to cloud resources
@@ -21,8 +21,8 @@ from cartography.config import Config as CartographyConfig
from celery.utils.log import get_task_logger
from prowler.config import config as ProwlerConfig
from tasks.jobs.attack_paths.config import (
BATCH_SIZE,
FINDINGS_BATCH_SIZE,
GRAPH_MUTATION_BATCH_SIZE,
get_node_uid_field,
get_provider_resource_label,
get_root_node_label,
@@ -135,7 +135,7 @@ def add_resource_label(
while labeled_count > 0:
result = neo4j_session.run(
query,
{"provider_uid": provider_uid, "batch_size": GRAPH_MUTATION_BATCH_SIZE},
{"provider_uid": provider_uid, "batch_size": BATCH_SIZE},
)
labeled_count = result.single().get("labeled_count", 0)
total_labeled += labeled_count
+12 -31
View File
@@ -372,19 +372,7 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
except Exception as e:
exception_message = utils.stringify_exception(e, "Attack Paths scan failed")
temporary_database_missing = (
isinstance(e, graph_database.GraphDatabaseQueryException)
and e.code == graph_database.DATABASE_NOT_FOUND_CODE
and tmp_database_name in str(e)
)
if temporary_database_missing:
logger.warning(exception_message)
else:
logger.exception(exception_message)
cleanup_log_level = (
logging.WARNING if temporary_database_missing else logging.ERROR
)
cleanup_exc_info = not temporary_database_missing
logger.exception(exception_message)
ingestion_exceptions["global_error"] = exception_message
# Recover `graph_data_ready` based on how far the swap got
@@ -399,24 +387,19 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
)
except Exception:
logger.log(
cleanup_log_level,
"Failed to recover `graph_data_ready` for provider "
f"{attack_paths_scan.provider_id}",
exc_info=cleanup_exc_info,
logger.error(
f"Failed to recover `graph_data_ready` for provider {attack_paths_scan.provider_id}",
exc_info=True,
)
# Dropping the temporary database if it still exists
try:
graph_database.drop_database(tmp_cartography_config.neo4j_database)
except Exception as cleanup_error:
logger.log(
cleanup_log_level,
"Failed to drop temporary Neo4j database "
f"`{tmp_cartography_config.neo4j_database}` during cleanup: "
f"{cleanup_error}",
exc_info=cleanup_exc_info,
except Exception as e:
logger.error(
f"Failed to drop temporary Neo4j database `{tmp_cartography_config.neo4j_database}` during cleanup: {e}",
exc_info=True,
)
# Set Attack Paths scan state to FAILED
@@ -424,12 +407,10 @@ def run(tenant_id: str, scan_id: str, task_id: str) -> dict[str, Any]:
db_utils.finish_attack_paths_scan(
attack_paths_scan, StateChoices.FAILED, ingestion_exceptions
)
except Exception as cleanup_error:
logger.log(
cleanup_log_level,
f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` "
f"(row may have been deleted): {cleanup_error}",
exc_info=cleanup_exc_info,
except Exception as e:
logger.error(
f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` (row may have been deleted): {e}",
exc_info=True,
)
raise
@@ -30,6 +30,7 @@ from tasks.jobs.attack_paths.config import (
PROVIDER_CONFIGS,
PROVIDER_ISOLATION_PROPERTIES,
PROVIDER_RESOURCE_LABEL,
SYNC_BATCH_SIZE,
NormalizedList,
get_provider_label,
get_tenant_label,
@@ -115,7 +116,6 @@ def sync_nodes(
Source and target sessions are opened sequentially per batch to avoid
holding two Bolt connections simultaneously for the entire sync duration.
"""
batch_size = sink.sync_batch_size
t0 = time.perf_counter()
last_id = -1
parents_synced = 0
@@ -137,7 +137,7 @@ def sync_nodes(
with graph_database.get_session(source_database) as source_session:
result = source_session.run(
NODE_FETCH_QUERY,
{"last_id": last_id, "batch_size": batch_size},
{"last_id": last_id, "batch_size": SYNC_BATCH_SIZE},
)
for record in result:
batch_count += 1
@@ -156,17 +156,17 @@ def sync_nodes(
for labels, batch in parent_groups.items():
rendered_labels = _render_labels(labels, extra_labels)
for sink_batch in _iter_sink_batches(batch, batch_size):
for sink_batch in _iter_sink_batches(batch):
sink.write_nodes(target_database, rendered_labels, sink_batch)
for child_label, batch in child_groups.items():
rendered_labels = _render_labels((child_label,), extra_labels)
for sink_batch in _iter_sink_batches(batch, batch_size):
for sink_batch in _iter_sink_batches(batch):
sink.write_nodes(target_database, rendered_labels, sink_batch)
children_synced += len(batch)
for rel_type, batch in rel_groups.items():
for sink_batch in _iter_sink_batches(batch, batch_size):
for sink_batch in _iter_sink_batches(batch):
sink.write_relationships(
target_database, rel_type, provider_id, sink_batch
)
@@ -205,7 +205,6 @@ def sync_relationships(
Source and target sessions are opened sequentially per batch to avoid
holding two Bolt connections simultaneously for the entire sync duration.
"""
batch_size = sink.sync_batch_size
t0 = time.perf_counter()
last_id = -1
total_synced = 0
@@ -218,7 +217,7 @@ def sync_relationships(
with graph_database.get_session(source_database) as source_session:
result = source_session.run(
RELATIONSHIPS_FETCH_QUERY,
{"last_id": last_id, "batch_size": batch_size},
{"last_id": last_id, "batch_size": SYNC_BATCH_SIZE},
)
for record in result:
batch_count += 1
@@ -230,7 +229,7 @@ def sync_relationships(
break
for rel_type, batch in grouped.items():
for sink_batch in _iter_sink_batches(batch, batch_size):
for sink_batch in _iter_sink_batches(batch):
sink.write_relationships(
target_database, rel_type, provider_id, sink_batch
)
@@ -248,9 +247,10 @@ def sync_relationships(
def _iter_sink_batches(
rows: list[dict[str, Any]],
batch_size: int,
batch_size: int | None = None,
) -> Iterator[list[dict[str, Any]]]:
"""Yield final sink write batches after source rows have been transformed."""
batch_size = SYNC_BATCH_SIZE if batch_size is None else batch_size
if batch_size <= 0:
raise ValueError("Sink batch size must be greater than zero")
+1 -8
View File
@@ -11,7 +11,6 @@ from api.compliance import (
from api.db_router import READ_REPLICA_ALIAS
from api.db_utils import delete_related_daily_task, rls_transaction
from api.decorators import handle_provider_deletion, set_tenant
from api.exceptions import ProviderDeletedException
from api.models import (
Finding,
Integration,
@@ -667,13 +666,7 @@ class AttackPathsScanRLSTask(RLSTask):
scan_id = kwargs.get("scan_id")
if tenant_id and scan_id:
if isinstance(exc, ProviderDeletedException):
logger.warning(
f"Attack paths scan task {task_id} stopped because its provider "
f"or tenant was deleted: {exc}"
)
else:
logger.error(f"Attack paths scan task {task_id} failed: {exc}")
logger.error(f"Attack paths scan task {task_id} failed: {exc}")
attack_paths_db_utils.fail_attack_paths_scan(tenant_id, scan_id, str(exc))
@@ -1,102 +0,0 @@
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import neo4j.exceptions
import pytest
from tasks.jobs.attack_paths import aws
DATABASE_NOT_FOUND_CODE = "Neo.ClientError.Database.DatabaseNotFound"
def _make_neo4j_error(code: str) -> neo4j.exceptions.Neo4jError:
return neo4j.exceptions.Neo4jError._hydrate_neo4j(
code=code,
message="graph query failed",
)
def _resource_functions(failing_sync, following_sync):
return {
"failing_sync": failing_sync,
"following_sync": following_sync,
"permission_relationships": MagicMock(),
"resourcegroupstaggingapi": MagicMock(),
}
def test_sync_aws_account_reraises_database_not_found_immediately():
error = _make_neo4j_error(DATABASE_NOT_FOUND_CODE)
failing_sync = MagicMock(side_effect=error)
following_sync = MagicMock()
with (
patch.object(
aws.cartography_aws,
"RESOURCE_FUNCTIONS",
_resource_functions(failing_sync, following_sync),
),
patch.object(aws.db_utils, "update_attack_paths_scan_progress"),
patch.object(aws.utils, "stringify_exception") as stringify_exception,
patch.object(aws.logger, "warning") as warning,
pytest.raises(neo4j.exceptions.Neo4jError) as exc_info,
):
aws.sync_aws_account(
SimpleNamespace(uid="123456789012"),
[
"failing_sync",
"following_sync",
"permission_relationships",
"resourcegroupstaggingapi",
],
{},
MagicMock(),
)
assert exc_info.value is error
following_sync.assert_not_called()
stringify_exception.assert_not_called()
warning.assert_not_called()
@pytest.mark.parametrize(
"error",
[
_make_neo4j_error("Neo.ClientError.Statement.SyntaxError"),
RuntimeError("resource sync failed"),
],
ids=["different-neo4j-error", "non-neo4j-error"],
)
def test_sync_aws_account_warns_and_continues_for_other_exceptions(error):
failing_sync = MagicMock(side_effect=error)
following_sync = MagicMock()
with (
patch.object(
aws.cartography_aws,
"RESOURCE_FUNCTIONS",
_resource_functions(failing_sync, following_sync),
),
patch.object(aws.db_utils, "update_attack_paths_scan_progress"),
patch.object(
aws.utils,
"stringify_exception",
return_value="formatted failure",
),
patch.object(aws.logger, "warning") as warning,
):
failed_syncs = aws.sync_aws_account(
SimpleNamespace(uid="123456789012"),
[
"failing_sync",
"following_sync",
"permission_relationships",
"resourcegroupstaggingapi",
],
{},
MagicMock(),
)
assert failed_syncs == {"failing_sync": "formatted failure"}
following_sync.assert_called_once_with()
warning.assert_called_once()
assert "Continuing to the next AWS sync function" in warning.call_args.args[0]
@@ -1,4 +1,3 @@
import logging
from contextlib import nullcontext
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
@@ -6,9 +5,7 @@ from unittest.mock import MagicMock, call, patch
from uuid import uuid4
import pytest
from api.attack_paths.database import GraphDatabaseQueryException
from api.db_utils import rls_transaction
from api.exceptions import ProviderDeletedException
from api.models import (
AttackPathsScan,
Finding,
@@ -253,32 +250,6 @@ class TestAttackPathsRun:
mock_starting.assert_not_called()
mock_create_db.assert_not_called()
@pytest.mark.parametrize(
("ingestion_error", "temporary_database_missing"),
[
(RuntimeError("ingestion boom"), False),
(
GraphDatabaseQueryException(
message="Graph not found: db-scan-id",
code="Neo.ClientError.Database.DatabaseNotFound",
),
True,
),
(
GraphDatabaseQueryException(
message="Graph not found: db-tenant-id",
code="Neo.ClientError.Database.DatabaseNotFound",
),
False,
),
],
ids=[
"regular-error",
"temporary-database-missing",
"sink-database-missing",
],
)
@patch("tasks.jobs.attack_paths.scan.logger")
@patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
return_value="Cartography failed: ingestion boom",
@@ -331,9 +302,6 @@ class TestAttackPathsRun:
mock_drop_db,
mock_event_loop,
mock_stringify,
mock_logger,
ingestion_error,
temporary_database_missing,
tenants_fixture,
aws_provider,
scans_fixture,
@@ -353,11 +321,7 @@ class TestAttackPathsRun:
session_ctx = MagicMock()
session_ctx.__enter__.return_value = mock_session
session_ctx.__exit__.return_value = False
ingestion_fn = MagicMock(side_effect=ingestion_error)
if temporary_database_missing:
mock_finish.side_effect = DatabaseError(
"Save with update_fields did not affect any rows"
)
ingestion_fn = MagicMock(side_effect=RuntimeError("ingestion boom"))
with (
patch(
@@ -373,28 +337,13 @@ class TestAttackPathsRun:
return_value=ingestion_fn,
),
):
with pytest.raises(type(ingestion_error)):
with pytest.raises(RuntimeError, match="ingestion boom"):
attack_paths_run(str(tenant.id), str(scan.id), "task-456")
failure_args = mock_finish.call_args[0]
assert failure_args[0] is attack_paths_scan
assert failure_args[1] == StateChoices.FAILED
assert failure_args[2] == {"global_error": "Cartography failed: ingestion boom"}
mock_drop_db.assert_called_once_with("db-scan-id")
if temporary_database_missing:
mock_logger.warning.assert_any_call("Cartography failed: ingestion boom")
mock_logger.exception.assert_not_called()
mock_logger.log.assert_called_once_with(
logging.WARNING,
f"Could not mark Attack Paths scan {attack_paths_scan.id} as `FAILED` "
"(row may have been deleted): Save with update_fields did not affect "
"any rows",
exc_info=False,
)
else:
mock_logger.exception.assert_called_once_with(
"Cartography failed: ingestion boom"
)
@patch(
"tasks.jobs.attack_paths.scan.utils.stringify_exception",
@@ -1316,33 +1265,6 @@ class TestAttackPathsScanRLSTaskOnFailure:
mock_fail.assert_called_once_with("t-1", "s-1", "boom")
def test_on_failure_logs_provider_deletion_as_warning(self):
from tasks.tasks import AttackPathsScanRLSTask
task = AttackPathsScanRLSTask()
error = ProviderDeletedException("provider deleted")
with (
patch("tasks.tasks.logger") as mock_logger,
patch(
"tasks.tasks.attack_paths_db_utils.fail_attack_paths_scan"
) as mock_fail,
):
task.on_failure(
exc=error,
task_id="task-abc",
args=(),
kwargs={"tenant_id": "t-1", "scan_id": "s-1"},
_einfo=None,
)
mock_logger.warning.assert_called_once_with(
"Attack paths scan task task-abc stopped because its provider or tenant "
"was deleted: provider deleted"
)
mock_logger.error.assert_not_called()
mock_fail.assert_called_once_with("t-1", "s-1", "provider deleted")
def test_on_failure_skips_when_missing_kwargs(self):
from tasks.tasks import AttackPathsScanRLSTask
@@ -1974,7 +1896,7 @@ class TestSyncNodes:
mock_source_1.run.return_value = [row]
mock_source_2 = MagicMock()
mock_source_2.run.return_value = []
sink = MagicMock(sync_batch_size=1000)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
@@ -2011,7 +1933,7 @@ class TestSyncNodes:
src_1.run.return_value = [row]
src_2 = MagicMock()
src_2.run.return_value = []
sink = MagicMock(sync_batch_size=1000)
sink = MagicMock()
sink.write_nodes.side_effect = lambda *_a, **_kw: call_order.append(
"sink:write"
)
@@ -2047,15 +1969,18 @@ class TestSyncNodes:
src_2.run.return_value = [row_b]
src_3 = MagicMock()
src_3.run.return_value = []
sink = MagicMock(sync_batch_size=1)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
_make_session_ctx(src_3),
],
with (
patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
_make_session_ctx(src_3),
],
),
patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 1),
):
result = sync_module.sync_nodes("src", "tgt", "t-1", "p-1", sink, [])
@@ -2084,14 +2009,17 @@ class TestSyncNodes:
src_1.run.return_value = [row]
src_2 = MagicMock()
src_2.run.return_value = []
sink = MagicMock(sync_batch_size=2)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
],
with (
patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
],
),
patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 2),
):
result = sync_module.sync_nodes(
"src", "tgt", "t-1", "p-1", sink, normalized_lists
@@ -2109,7 +2037,7 @@ class TestSyncNodes:
def test_sync_nodes_empty_source_returns_zero(self):
src = MagicMock()
src.run.return_value = []
sink = MagicMock(sync_batch_size=1000)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
@@ -2138,7 +2066,7 @@ class TestSyncRelationships:
src_1.run.return_value = [row]
src_2 = MagicMock()
src_2.run.return_value = []
sink = MagicMock(sync_batch_size=1000)
sink = MagicMock()
sink.write_relationships.side_effect = lambda *_a, **_kw: call_order.append(
"sink:write"
)
@@ -2176,15 +2104,18 @@ class TestSyncRelationships:
src_2.run.return_value = [row_b]
src_3 = MagicMock()
src_3.run.return_value = []
sink = MagicMock(sync_batch_size=1)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
_make_session_ctx(src_3),
],
with (
patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
_make_session_ctx(src_3),
],
),
patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 1),
):
total = sync_module.sync_relationships("src", "tgt", "p-1", sink)
@@ -2209,14 +2140,17 @@ class TestSyncRelationships:
src_1.run.return_value = rows
src_2 = MagicMock()
src_2.run.return_value = []
sink = MagicMock(sync_batch_size=2)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
],
with (
patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
side_effect=[
_make_session_ctx(src_1),
_make_session_ctx(src_2),
],
),
patch("tasks.jobs.attack_paths.sync.SYNC_BATCH_SIZE", 2),
):
total = sync_module.sync_relationships("src", "tgt", "p-1", sink)
@@ -2229,7 +2163,7 @@ class TestSyncRelationships:
def test_sync_relationships_empty_source_returns_zero(self):
src = MagicMock()
src.run.return_value = []
sink = MagicMock(sync_batch_size=1000)
sink = MagicMock()
with patch(
"tasks.jobs.attack_paths.sync.graph_database.get_session",
@@ -3122,61 +3056,6 @@ class TestCleanupStaleAttackPathsScans:
ap_scan.refresh_from_db()
assert ap_scan.state == StateChoices.FAILED
@pytest.mark.parametrize(
("age_seconds", "should_clean"),
[
(960 * 60 - 1, False),
(960 * 60, False),
(960 * 60 + 1, True),
],
)
@patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready")
@patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database")
@patch(
"tasks.jobs.attack_paths.cleanup.rls_transaction",
new=lambda *args, **kwargs: nullcontext(),
)
@patch("tasks.jobs.attack_paths.cleanup._revoke_task")
@patch("tasks.jobs.attack_paths.cleanup._ping_workers")
def test_stale_threshold_boundary_is_strict(
self,
mock_ping,
mock_revoke,
mock_drop_db,
mock_recover,
age_seconds,
should_clean,
tenants_fixture,
aws_provider,
):
from tasks.jobs.attack_paths.cleanup import cleanup_stale_attack_paths_scans
now = datetime.now(tz=UTC)
ap_scan, task_result = self._create_executing_scan(
tenants_fixture[0],
aws_provider,
started_at=now - timedelta(seconds=age_seconds),
worker="live-worker@host",
)
mock_ping.return_value = ({"live-worker@host"}, set())
with patch("tasks.jobs.attack_paths.cleanup.datetime") as mock_datetime:
mock_datetime.now.return_value = now
result = cleanup_stale_attack_paths_scans()
assert result["cleaned_up_count"] == int(should_clean)
ap_scan.refresh_from_db()
expected_state = StateChoices.FAILED if should_clean else StateChoices.EXECUTING
assert ap_scan.state == expected_state
if should_clean:
mock_revoke.assert_called_once_with(task_result, terminate=True)
mock_drop_db.assert_called_once()
mock_recover.assert_called_once()
else:
mock_revoke.assert_not_called()
mock_drop_db.assert_not_called()
mock_recover.assert_not_called()
@patch("tasks.jobs.attack_paths.cleanup.recover_graph_data_ready")
@patch("tasks.jobs.attack_paths.cleanup.graph_database.drop_database")
@patch(
@@ -3304,7 +3304,6 @@ class TestTaskTimeLimits:
for name in (
"scan-perform",
"scan-perform-scheduled",
"attack-paths-scan-perform",
"provider-deletion",
"tenant-deletion",
):
Generated
+4 -4
View File
@@ -4673,8 +4673,8 @@ wheels = [
[[package]]
name = "prowler"
version = "5.34.0"
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=v5.34#3fd9fca32f41c79b7a6e672b1d4eac56c732f1c0" }
version = "5.32.0"
source = { git = "https://github.com/prowler-cloud/prowler.git?rev=master#5dac8a0a53272e4db68c476fb969dc03e88beb68" }
dependencies = [
{ name = "alibabacloud-actiontrail20200706" },
{ name = "alibabacloud-credentials" },
@@ -4762,7 +4762,7 @@ dependencies = [
[[package]]
name = "prowler-api"
version = "1.35.1"
version = "1.36.0"
source = { virtual = "." }
dependencies = [
{ name = "cartography" },
@@ -4862,7 +4862,7 @@ requires-dist = [
{ name = "matplotlib", specifier = "==3.10.8" },
{ name = "neo4j", specifier = "==6.1.0" },
{ name = "openai", specifier = "==1.109.1" },
{ name = "prowler", git = "https://github.com/prowler-cloud/prowler.git?rev=v5.34" },
{ name = "prowler", git = "https://github.com/prowler-cloud/prowler.git?rev=master" },
{ name = "psycopg2-binary", specifier = "==2.9.9" },
{ name = "pytest-celery", extras = ["redis"], specifier = "==1.3.0" },
{ name = "reportlab", specifier = "==4.4.10" },
@@ -128,8 +128,8 @@ To update the environment file:
Edit the `.env` file and change version values:
```env
PROWLER_UI_VERSION="5.33.0"
PROWLER_API_VERSION="5.33.0"
PROWLER_UI_VERSION="5.34.0"
PROWLER_API_VERSION="5.34.0"
```
<Note>
@@ -1 +0,0 @@
Jira tenant information requests validate site names and do not follow redirects
+1 -1
View File
@@ -49,7 +49,7 @@ class _MutableTimestamp:
timestamp = _MutableTimestamp(datetime.today())
timestamp_utc = _MutableTimestamp(datetime.now(timezone.utc))
prowler_version = "5.34.1"
prowler_version = "5.35.0"
html_logo_url = "https://github.com/prowler-cloud/prowler/"
square_logo_img = "https://raw.githubusercontent.com/prowler-cloud/prowler/dc7d2d5aeb92fdf12e8604f42ef6472cd3e8e889/docs/img/prowler-logo-black.png"
aws_logo = "https://user-images.githubusercontent.com/38561120/235953920-3e3fba08-0795-41dc-b480-9bea57db9f2e.png"
-8
View File
@@ -1,6 +1,5 @@
import base64
import os
import re
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Dict, List, Optional
@@ -38,10 +37,6 @@ from prowler.lib.outputs.jira.exceptions.exceptions import (
)
from prowler.providers.common.models import Connection
ATLASSIAN_SITE_NAME_REGEX = re.compile(
r"\A[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\Z"
)
@dataclass
class JiraConnection(Connection):
@@ -670,14 +665,11 @@ class Jira:
"""
try:
if self._using_basic_auth:
if not domain or not ATLASSIAN_SITE_NAME_REGEX.fullmatch(domain):
raise ValueError("Invalid Jira site name.")
headers = self.get_headers(access_token)
response = requests.get(
f"https://{domain}.atlassian.net/_edge/tenant_info",
headers=headers,
timeout=self.REQUEST_TIMEOUT,
allow_redirects=False,
)
response = response.json()
return response.get("cloudId")
+1 -1
View File
@@ -125,7 +125,7 @@ maintainers = [{name = "Prowler Engineering", email = "engineering@prowler.com"}
name = "prowler"
readme = "README.md"
requires-python = ">=3.10,<3.14"
version = "5.34.1"
version = "5.35.0"
[project.scripts]
prowler = "prowler.__main__:prowler"
@@ -223,6 +223,7 @@ FINDINGS_TABLE_PARTITION_MAX_AGE_MONTHS = env.int("...", None) # Optional clean
| `DJANGO_DELETION_BATCH_SIZE` | `5000` | Batch size for deletions |
| `DJANGO_LOGGING_LEVEL` | `INFO` | Log level |
| `DJANGO_LOGGING_FORMATTER` | `ndjson` | Log format (`ndjson` or `human_readable`) |
| `DJANGO_MIGRATION_LOCK_TIMEOUT` | `5s` | `lock_timeout` for the `migrate` command only; `0` disables it |
---
+1 -30
View File
@@ -56,7 +56,7 @@ class TestJiraIntegration:
self.user_mail = "test_user_mail"
self.api_token = "test_api_token"
self.domain = "test-domain"
self.domain = "test_domain"
self.jira_integration_basic_auth = Jira(
user_mail=self.user_mail,
@@ -386,35 +386,6 @@ class TestJiraIntegration:
assert mock_get.call_args.kwargs["timeout"] == Jira.REQUEST_TIMEOUT
@pytest.mark.parametrize(
"domain",
(
"169.254.169.254#",
"internal/service",
"internal?target",
"internal\\target",
"internal:8000",
"user@internal",
),
)
@patch("prowler.lib.outputs.jira.jira.requests.get")
def test_get_cloud_id_basic_auth_rejects_invalid_domain(self, mock_get, domain):
with pytest.raises(JiraGetCloudIDError):
self.jira_integration_basic_auth.get_cloud_id(domain=domain)
mock_get.assert_not_called()
@patch("prowler.lib.outputs.jira.jira.requests.get")
def test_get_cloud_id_basic_auth_disables_redirects(self, mock_get):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"cloudId": "test_cloud_id"}
mock_get.return_value = mock_response
self.jira_integration_basic_auth.get_cloud_id(domain=self.domain)
assert mock_get.call_args.kwargs["allow_redirects"] is False
@patch("prowler.lib.outputs.jira.jira.requests.post")
def test_refresh_access_token_sends_timeout(self, mock_post):
"""refresh_access_token must pass a request timeout."""
Generated
+1 -1
View File
@@ -3553,7 +3553,7 @@ wheels = [
[[package]]
name = "prowler"
version = "5.34.1"
version = "5.35.0"
source = { editable = "." }
dependencies = [
{ name = "alibabacloud-actiontrail20200706" },