diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 59998e9fba..1b25229f3b 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -5,9 +5,16 @@ All notable changes to the **Prowler API** are documented in this file. ## [v1.10.0] (Prowler UNRELEASED) ### Added - - SSO with SAML support [(#8175)](https://github.com/prowler-cloud/prowler/pull/8175) +- `GET /resources/metadata`, `GET /resources/metadata/latest` and `GET /resources/latest` to expose resource metadata and latest scan results [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112) + +### Changed - `/processors` endpoints to post-process findings. Currently, only the Mutelist processor is supported to allow to mute findings. +- Optimized the underlying queries for resources endpoints [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112) +- Optimized include parameters for resources view [(#8229)](https://github.com/prowler-cloud/prowler/pull/8229) + +### Fixed +- Search filter for findings and resources [(#8112)](https://github.com/prowler-cloud/prowler/pull/8112) ### Security diff --git a/api/pyproject.toml b/api/pyproject.toml index b01e2cda14..2214df5a8f 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -38,7 +38,7 @@ name = "prowler-api" package-mode = false # Needed for the SDK compatibility requires-python = ">=3.11,<3.13" -version = "1.9.0" +version = "1.10.0" [project.scripts] celery = "src.backend.config.settings.celery" diff --git a/api/src/backend/api/adapters.py b/api/src/backend/api/adapters.py index bf3fb54655..e09dc972b4 100644 --- a/api/src/backend/api/adapters.py +++ b/api/src/backend/api/adapters.py @@ -65,5 +65,7 @@ class ProwlerSocialAccountAdapter(DefaultSocialAccountAdapter): role=role, tenant_id=tenant.id, ) + else: + request.session["saml_user_created"] = str(user.id) return user diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index ee69359000..7418d2daf9 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -1,5 +1,6 @@ from datetime import date, datetime, timedelta, timezone +from dateutil.parser import parse from django.conf import settings from django.db.models import Q from django_filters.rest_framework import ( @@ -339,6 +340,8 @@ class ResourceFilter(ProviderRelationshipFilterSet): tags = CharFilter(method="filter_tag") inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") updated_at = DateFilter(field_name="updated_at", lookup_expr="date") + scan = UUIDFilter(field_name="provider__scan", lookup_expr="exact") + scan__in = UUIDInFilter(field_name="provider__scan", lookup_expr="in") class Meta: model = Resource @@ -353,6 +356,82 @@ class ResourceFilter(ProviderRelationshipFilterSet): "updated_at": ["gte", "lte"], } + def filter_queryset(self, queryset): + if not (self.data.get("scan") or self.data.get("scan__in")) and not ( + self.data.get("updated_at") + or self.data.get("updated_at__date") + or self.data.get("updated_at__gte") + or self.data.get("updated_at__lte") + ): + raise ValidationError( + [ + { + "detail": "At least one date filter is required: filter[updated_at], filter[updated_at.gte], " + "or filter[updated_at.lte].", + "status": 400, + "source": {"pointer": "/data/attributes/updated_at"}, + "code": "required", + } + ] + ) + + gte_date = ( + parse(self.data.get("updated_at__gte")).date() + if self.data.get("updated_at__gte") + else datetime.now(timezone.utc).date() + ) + lte_date = ( + parse(self.data.get("updated_at__lte")).date() + if self.data.get("updated_at__lte") + else datetime.now(timezone.utc).date() + ) + + if abs(lte_date - gte_date) > timedelta( + days=settings.FINDINGS_MAX_DAYS_IN_RANGE + ): + raise ValidationError( + [ + { + "detail": f"The date range cannot exceed {settings.FINDINGS_MAX_DAYS_IN_RANGE} days.", + "status": 400, + "source": {"pointer": "/data/attributes/updated_at"}, + "code": "invalid", + } + ] + ) + + return super().filter_queryset(queryset) + + def filter_tag_key(self, queryset, name, value): + return queryset.filter(Q(tags__key=value) | Q(tags__key__icontains=value)) + + def filter_tag_value(self, queryset, name, value): + return queryset.filter(Q(tags__value=value) | Q(tags__value__icontains=value)) + + def filter_tag(self, queryset, name, value): + # We won't know what the user wants to filter on just based on the value, + # and we don't want to build special filtering logic for every possible + # provider tag spec, so we'll just do a full text search + return queryset.filter(tags__text_search=value) + + +class LatestResourceFilter(ProviderRelationshipFilterSet): + tag_key = CharFilter(method="filter_tag_key") + tag_value = CharFilter(method="filter_tag_value") + tag = CharFilter(method="filter_tag") + tags = CharFilter(method="filter_tag") + + class Meta: + model = Resource + fields = { + "provider": ["exact", "in"], + "uid": ["exact", "icontains"], + "name": ["exact", "icontains"], + "region": ["exact", "icontains", "in"], + "service": ["exact", "icontains", "in"], + "type": ["exact", "icontains", "in"], + } + def filter_tag_key(self, queryset, name, value): return queryset.filter(Q(tags__key=value) | Q(tags__key__icontains=value)) diff --git a/api/src/backend/api/migrations/0036_rfm_tenant_finding_index_partitions.py b/api/src/backend/api/migrations/0036_rfm_tenant_finding_index_partitions.py new file mode 100644 index 0000000000..cd360eb7e4 --- /dev/null +++ b/api/src/backend/api/migrations/0036_rfm_tenant_finding_index_partitions.py @@ -0,0 +1,30 @@ +from functools import partial + +from django.db import migrations + +from api.db_utils import create_index_on_partitions, drop_index_on_partitions + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0035_finding_muted_reason"), + ] + + operations = [ + migrations.RunPython( + partial( + create_index_on_partitions, + parent_table="resource_finding_mappings", + index_name="rfm_tenant_finding_idx", + columns="tenant_id, finding_id", + method="BTREE", + ), + reverse_code=partial( + drop_index_on_partitions, + parent_table="resource_finding_mappings", + index_name="rfm_tenant_finding_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0037_rfm_tenant_finding_index_parent.py b/api/src/backend/api/migrations/0037_rfm_tenant_finding_index_parent.py new file mode 100644 index 0000000000..178cf900f6 --- /dev/null +++ b/api/src/backend/api/migrations/0037_rfm_tenant_finding_index_parent.py @@ -0,0 +1,17 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0036_rfm_tenant_finding_index_partitions"), + ] + + operations = [ + migrations.AddIndex( + model_name="resourcefindingmapping", + index=models.Index( + fields=["tenant_id", "finding_id"], + name="rfm_tenant_finding_idx", + ), + ), + ] diff --git a/api/src/backend/api/migrations/0038_resource_failed_findings_count.py b/api/src/backend/api/migrations/0038_resource_failed_findings_count.py new file mode 100644 index 0000000000..7bf9b4c856 --- /dev/null +++ b/api/src/backend/api/migrations/0038_resource_failed_findings_count.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0037_rfm_tenant_finding_index_parent"), + ] + + operations = [ + migrations.AddField( + model_name="resource", + name="failed_findings_count", + field=models.IntegerField(default=0), + ) + ] diff --git a/api/src/backend/api/migrations/0039_resource_resources_failed_findings_idx.py b/api/src/backend/api/migrations/0039_resource_resources_failed_findings_idx.py new file mode 100644 index 0000000000..2c2ac3ce31 --- /dev/null +++ b/api/src/backend/api/migrations/0039_resource_resources_failed_findings_idx.py @@ -0,0 +1,20 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("api", "0038_resource_failed_findings_count"), + ] + + operations = [ + AddIndexConcurrently( + model_name="resource", + index=models.Index( + fields=["tenant_id", "-failed_findings_count", "id"], + name="resources_failed_findings_idx", + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index 0d486afed3..f81ebafaf3 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -561,6 +561,8 @@ class Resource(RowLevelSecurityProtectedModel): details = models.TextField(blank=True, null=True) partition = models.TextField(blank=True, null=True) + failed_findings_count = models.IntegerField(default=0) + # Relationships tags = models.ManyToManyField( ResourceTag, @@ -607,6 +609,10 @@ class Resource(RowLevelSecurityProtectedModel): fields=["tenant_id", "provider_id"], name="resources_tenant_provider_idx", ), + models.Index( + fields=["tenant_id", "-failed_findings_count", "id"], + name="resources_failed_findings_idx", + ), ] constraints = [ @@ -849,6 +855,12 @@ class ResourceFindingMapping(PostgresPartitionedModel, RowLevelSecurityProtected # - tenant_id # - id + indexes = [ + models.Index( + fields=["tenant_id", "finding_id"], + name="rfm_tenant_finding_idx", + ), + ] constraints = [ models.UniqueConstraint( fields=("tenant_id", "resource_id", "finding_id"), diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 0164b63b7c..7f93baeb9a 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: Prowler API - version: 1.9.0 + version: 1.10.0 description: |- Prowler API specification. @@ -4677,6 +4677,7 @@ paths: - tags - provider - findings + - failed_findings_count - url - type description: endpoint return only specific fields in the response on a per-type @@ -4809,6 +4810,365 @@ paths: description: Multiple values may be separated by commas. explode: false style: form + - in: query + name: filter[scan] + schema: + type: string + format: uuid + - in: query + name: filter[scan__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[tag] + schema: + type: string + - in: query + name: filter[tag_key] + schema: + type: string + - in: query + name: filter[tag_value] + schema: + type: string + - in: query + name: filter[tags] + schema: + type: string + - in: query + name: filter[type] + schema: + type: string + - in: query + name: filter[type__icontains] + schema: + type: string + - in: query + name: filter[type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[uid__icontains] + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date + description: At least one of the variations of the `filter[updated_at]` filter + must be provided. + required: true + - in: query + name: filter[updated_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[updated_at__lte] + schema: + type: string + format: date-time + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - findings + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - provider_uid + - -provider_uid + - uid + - -uid + - name + - -name + - region + - -region + - service + - -service + - type + - -type + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Resource + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedResourceList' + description: '' + /api/v1/resources/{id}: + get: + operationId: resources_retrieve + description: Fetch detailed information about a specific resource by their ID. + A Resource is an object that is discovered by Prowler. It can be anything + from a single host to a whole VPC. + summary: Retrieve data for a resource + parameters: + - in: query + name: fields[resources] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - uid + - name + - region + - service + - tags + - provider + - findings + - failed_findings_count + - url + - type + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this resource. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - findings + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Resource + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ResourceResponse' + description: '' + /api/v1/resources/latest: + get: + operationId: resources_latest_retrieve + description: Retrieve a list of the latest resources from the latest scans for + each provider with options for filtering by various criteria. + summary: List the latest resources + parameters: + - in: query + name: fields[resources] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - uid + - name + - region + - service + - tags + - provider + - findings + - failed_findings_count + - url + - type + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[name] + schema: + type: string + - in: query + name: filter[name__icontains] + schema: + type: string + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form - name: filter[search] required: false in: query @@ -4873,21 +5233,6 @@ paths: name: filter[uid__icontains] schema: type: string - - in: query - name: filter[updated_at] - schema: - type: string - format: date - - in: query - name: filter[updated_at__gte] - schema: - type: string - format: date-time - - in: query - name: filter[updated_at__lte] - schema: - type: string - format: date-time - in: query name: include schema: @@ -4900,18 +5245,6 @@ paths: description: include query parameter to allow the client to customize which related resources should be returned. explode: false - - name: page[number] - required: false - in: query - description: A page number within the paginated result set. - schema: - type: integer - - name: page[size] - required: false - in: query - description: Number of results to return per page. - schema: - type: integer - name: sort required: false in: query @@ -4947,55 +5280,277 @@ paths: content: application/vnd.api+json: schema: - $ref: '#/components/schemas/PaginatedResourceList' + $ref: '#/components/schemas/ResourceResponse' description: '' - /api/v1/resources/{id}: + /api/v1/resources/metadata: get: - operationId: resources_retrieve - description: Fetch detailed information about a specific resource by their ID. - A Resource is an object that is discovered by Prowler. It can be anything - from a single host to a whole VPC. - summary: Retrieve data for a resource + operationId: resources_metadata_retrieve + description: Fetch unique metadata values from a set of resources. This is useful + for dynamic filtering. + summary: Retrieve metadata values from resources parameters: - in: query - name: fields[resources] + name: fields[resources-metadata] schema: type: array items: type: string enum: - - inserted_at - - updated_at - - uid - - name - - region - - service - - tags - - provider - - findings - - url - - type + - services + - regions + - types description: endpoint return only specific fields in the response on a per-type basis by including a fields[TYPE] query parameter. explode: false - - in: path - name: id + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[name] + schema: + type: string + - in: query + name: filter[name__icontains] + schema: + type: string + - in: query + name: filter[provider] schema: type: string format: uuid - description: A UUID string identifying this resource. - required: true - in: query - name: include + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + - in: query + name: filter[provider_type__in] schema: type: array items: type: string enum: - - findings - - provider - description: include query parameter to allow the client to customize which - related resources should be returned. + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[scan] + schema: + type: string + format: uuid + - in: query + name: filter[scan__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[tag] + schema: + type: string + - in: query + name: filter[tag_key] + schema: + type: string + - in: query + name: filter[tag_value] + schema: + type: string + - in: query + name: filter[tags] + schema: + type: string + - in: query + name: filter[type] + schema: + type: string + - in: query + name: filter[type__icontains] + schema: + type: string + - in: query + name: filter[type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[uid__icontains] + schema: + type: string + - in: query + name: filter[updated_at] + schema: + type: string + format: date + description: At least one of the variations of the `filter[updated_at]` filter + must be provided. + required: true + - in: query + name: filter[updated_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[updated_at__lte] + schema: + type: string + format: date-time + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - provider_uid + - -provider_uid + - uid + - -uid + - name + - -name + - region + - -region + - service + - -service + - type + - -type + - inserted_at + - -inserted_at + - updated_at + - -updated_at explode: false tags: - Resource @@ -5006,7 +5561,240 @@ paths: content: application/vnd.api+json: schema: - $ref: '#/components/schemas/ResourceResponse' + $ref: '#/components/schemas/ResourceMetadataResponse' + description: '' + /api/v1/resources/metadata/latest: + get: + operationId: resources_metadata_latest_retrieve + description: Fetch unique metadata values from a set of resources from the latest + scans for each provider. This is useful for dynamic filtering. + summary: Retrieve metadata values from the latest resources + parameters: + - in: query + name: fields[resources-metadata] + schema: + type: array + items: + type: string + enum: + - services + - regions + - types + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[name] + schema: + type: string + - in: query + name: filter[name__icontains] + schema: + type: string + - in: query + name: filter[provider] + schema: + type: string + format: uuid + - in: query + name: filter[provider__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_alias] + schema: + type: string + - in: query + name: filter[provider_alias__icontains] + schema: + type: string + - in: query + name: filter[provider_alias__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + - m365 + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + explode: false + style: form + - in: query + name: filter[provider_uid] + schema: + type: string + - in: query + name: filter[provider_uid__icontains] + schema: + type: string + - in: query + name: filter[provider_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[service] + schema: + type: string + - in: query + name: filter[service__icontains] + schema: + type: string + - in: query + name: filter[service__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[tag] + schema: + type: string + - in: query + name: filter[tag_key] + schema: + type: string + - in: query + name: filter[tag_value] + schema: + type: string + - in: query + name: filter[tags] + schema: + type: string + - in: query + name: filter[type] + schema: + type: string + - in: query + name: filter[type__icontains] + schema: + type: string + - in: query + name: filter[type__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[uid] + schema: + type: string + - in: query + name: filter[uid__icontains] + schema: + type: string + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - provider_uid + - -provider_uid + - uid + - -uid + - name + - -name + - region + - -region + - service + - -service + - type + - -type + - inserted_at + - -inserted_at + - updated_at + - -updated_at + explode: false + tags: + - Resource + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/ResourceMetadataResponse' description: '' /api/v1/roles: get: @@ -12601,6 +13389,9 @@ components: env: prod owner: johndoe readOnly: true + failed_findings_count: + type: integer + readOnly: true type: type: string readOnly: true @@ -12640,33 +13431,72 @@ components: type: object properties: data: - type: array - items: - type: object - properties: - id: - type: string - format: uuid - title: Resource Identifier - description: The identifier of the related object. - type: - type: string - enum: - - findings - title: Resource Type Name - description: The [type](https://jsonapi.org/format/#document-resource-object-identification) - member is used to describe resource objects that share common - attributes and relationships. - required: - - id - - type + type: object + properties: + id: + type: string + type: + type: string + enum: + - findings + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type required: - data - description: A related resource object from type findings - title: findings + description: The identifier of the related object. + title: Resource Identifier readOnly: true required: - provider + ResourceMetadata: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/ResourceMetadataTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + services: + type: array + items: + type: string + regions: + type: array + items: + type: string + types: + type: array + items: + type: string + required: + - services + - regions + - types + ResourceMetadataResponse: + type: object + properties: + data: + $ref: '#/components/schemas/ResourceMetadata' + required: + - data + ResourceMetadataTypeEnum: + type: string + enum: + - resources-metadata ResourceResponse: type: object properties: diff --git a/api/src/backend/api/tests/test_adapters.py b/api/src/backend/api/tests/test_adapters.py index 4351e3de3c..22b44b3506 100644 --- a/api/src/backend/api/tests/test_adapters.py +++ b/api/src/backend/api/tests/test_adapters.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from allauth.socialaccount.models import SocialLogin @@ -54,3 +54,24 @@ class TestProwlerSocialAccountAdapter: adapter.pre_social_login(rf.get("/"), sociallogin) sociallogin.connect.assert_not_called() + + def test_save_user_saml_sets_session_flag(self, rf): + adapter = ProwlerSocialAccountAdapter() + request = rf.get("/") + request.session = {} + + sociallogin = MagicMock(spec=SocialLogin) + sociallogin.provider = MagicMock() + sociallogin.provider.id = "saml" + sociallogin.account = MagicMock() + sociallogin.account.extra_data = {} + + mock_user = MagicMock() + mock_user.id = 123 + + with patch("api.adapters.super") as mock_super: + with patch("api.adapters.transaction"): + with patch("api.adapters.MainRouter"): + mock_super.return_value.save_user.return_value = mock_user + adapter.save_user(request, sociallogin) + assert request.session["saml_user_created"] == "123" diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index b064ee352f..2d59cffddd 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -2966,12 +2966,21 @@ class TestTaskViewSet: @pytest.mark.django_db class TestResourceViewSet: def test_resources_list_none(self, authenticated_client): - response = authenticated_client.get(reverse("resource-list")) + response = authenticated_client.get( + reverse("resource-list"), {"filter[updated_at]": TODAY} + ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == 0 - def test_resources_list(self, authenticated_client, resources_fixture): + def test_resources_list_no_date_filter(self, authenticated_client): response = authenticated_client.get(reverse("resource-list")) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json()["errors"][0]["code"] == "required" + + def test_resources_list(self, authenticated_client, resources_fixture): + response = authenticated_client.get( + reverse("resource-list"), {"filter[updated_at]": TODAY} + ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(resources_fixture) @@ -2992,7 +3001,8 @@ class TestResourceViewSet: findings_fixture, ): response = authenticated_client.get( - reverse("resource-list"), {"include": include_values} + reverse("resource-list"), + {"include": include_values, "filter[updated_at]": TODAY}, ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == len(resources_fixture) @@ -3020,8 +3030,9 @@ class TestResourceViewSet: ("region.icontains", "west", 1), ("service", "ec2", 2), ("service.icontains", "ec", 2), - ("inserted_at.gte", "2024-01-01 00:00:00", 3), - ("updated_at.lte", "2024-01-01 00:00:00", 0), + ("inserted_at.gte", today_after_n_days(-1), 3), + ("updated_at.gte", today_after_n_days(-1), 3), + ("updated_at.lte", today_after_n_days(1), 3), ("type.icontains", "prowler", 2), # provider filters ("provider_type", "aws", 3), @@ -3041,7 +3052,8 @@ class TestResourceViewSet: ("tags", "multi word", 1), # full text search on resource ("search", "arn", 3), - ("search", "def1", 1), + # To improve search efficiency, full text search is not fully applicable + # ("search", "def1", 1), # full text search on resource tags ("search", "multi word", 1), ("search", "key2", 2), @@ -3056,14 +3068,42 @@ class TestResourceViewSet: filter_value, expected_count, ): + filters = {f"filter[{filter_name}]": filter_value} + if "updated_at" not in filter_name: + filters["filter[updated_at]"] = TODAY response = authenticated_client.get( reverse("resource-list"), - {f"filter[{filter_name}]": filter_value}, + filters, ) assert response.status_code == status.HTTP_200_OK assert len(response.json()["data"]) == expected_count + def test_resource_filter_by_scan_id( + self, authenticated_client, resources_fixture, scans_fixture + ): + response = authenticated_client.get( + reverse("resource-list"), + {"filter[scan]": scans_fixture[0].id}, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + + def test_resource_filter_by_scan_id_in( + self, authenticated_client, resources_fixture, scans_fixture + ): + response = authenticated_client.get( + reverse("resource-list"), + { + "filter[scan.in]": [ + scans_fixture[0].id, + scans_fixture[1].id, + ] + }, + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 2 + def test_resource_filter_by_provider_id_in( self, authenticated_client, resources_fixture ): @@ -3073,7 +3113,8 @@ class TestResourceViewSet: "filter[provider.in]": [ resources_fixture[0].provider.id, resources_fixture[1].provider.id, - ] + ], + "filter[updated_at]": TODAY, }, ) assert response.status_code == status.HTTP_200_OK @@ -3110,13 +3151,13 @@ class TestResourceViewSet: ) def test_resources_sort(self, authenticated_client, sort_field): response = authenticated_client.get( - reverse("resource-list"), {"sort": sort_field} + reverse("resource-list"), {"filter[updated_at]": TODAY, "sort": sort_field} ) assert response.status_code == status.HTTP_200_OK def test_resources_sort_invalid(self, authenticated_client): response = authenticated_client.get( - reverse("resource-list"), {"sort": "invalid"} + reverse("resource-list"), {"filter[updated_at]": TODAY, "sort": "invalid"} ) assert response.status_code == status.HTTP_400_BAD_REQUEST assert response.json()["errors"][0]["code"] == "invalid" @@ -3149,6 +3190,100 @@ class TestResourceViewSet: ) assert response.status_code == status.HTTP_404_NOT_FOUND + def test_resources_metadata_retrieve( + self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture + ): + resource_1, *_ = resources_fixture + response = authenticated_client.get( + reverse("resource-metadata"), + {"filter[updated_at]": resource_1.updated_at.strftime("%Y-%m-%d")}, + ) + data = response.json() + + expected_services = {"ec2", "s3"} + expected_regions = {"us-east-1", "eu-west-1"} + expected_resource_types = {"prowler-test"} + + assert data["data"]["type"] == "resources-metadata" + assert data["data"]["id"] is None + assert set(data["data"]["attributes"]["services"]) == expected_services + assert set(data["data"]["attributes"]["regions"]) == expected_regions + assert set(data["data"]["attributes"]["types"]) == expected_resource_types + + def test_resources_metadata_resource_filter_retrieve( + self, authenticated_client, resources_fixture, backfill_scan_metadata_fixture + ): + resource_1, *_ = resources_fixture + response = authenticated_client.get( + reverse("resource-metadata"), + { + "filter[region]": "eu-west-1", + "filter[updated_at]": resource_1.updated_at.strftime("%Y-%m-%d"), + }, + ) + data = response.json() + + expected_services = {"s3"} + expected_regions = {"eu-west-1"} + expected_resource_types = {"prowler-test"} + + assert data["data"]["type"] == "resources-metadata" + assert data["data"]["id"] is None + assert set(data["data"]["attributes"]["services"]) == expected_services + assert set(data["data"]["attributes"]["regions"]) == expected_regions + assert set(data["data"]["attributes"]["types"]) == expected_resource_types + + def test_resources_metadata_future_date(self, authenticated_client): + response = authenticated_client.get( + reverse("resource-metadata"), + {"filter[updated_at]": "2048-01-01"}, + ) + data = response.json() + assert data["data"]["type"] == "resources-metadata" + assert data["data"]["id"] is None + assert data["data"]["attributes"]["services"] == [] + assert data["data"]["attributes"]["regions"] == [] + assert data["data"]["attributes"]["types"] == [] + + def test_resources_metadata_invalid_date(self, authenticated_client): + response = authenticated_client.get( + reverse("resource-metadata"), + {"filter[updated_at]": "2048-01-011"}, + ) + assert response.json() == { + "errors": [ + { + "detail": "Enter a valid date.", + "status": "400", + "source": {"pointer": "/data/attributes/updated_at"}, + "code": "invalid", + } + ] + } + + def test_resources_latest(self, authenticated_client, latest_scan_resource): + response = authenticated_client.get( + reverse("resource-latest"), + ) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["data"]) == 1 + assert ( + response.json()["data"][0]["attributes"]["uid"] == latest_scan_resource.uid + ) + + def test_resources_metadata_latest( + self, authenticated_client, latest_scan_resource + ): + response = authenticated_client.get( + reverse("resource-metadata_latest"), + ) + assert response.status_code == status.HTTP_200_OK + attributes = response.json()["data"]["attributes"] + + assert attributes["services"] == [latest_scan_resource.service] + assert attributes["regions"] == [latest_scan_resource.region] + assert attributes["types"] == [latest_scan_resource.type] + @pytest.mark.django_db class TestFindingViewSet: @@ -3247,7 +3382,7 @@ class TestFindingViewSet: ("search", "dev-qa", 1), ("search", "orange juice", 1), # full text search on resource - ("search", "ec2", 2), + ("search", "ec2", 1), # full text search on finding tags (disabled for now) # ("search", "value2", 2), # Temporary disabled until we implement tag filtering in the UI @@ -5984,6 +6119,7 @@ class TestTenantFinishACSView: reverse("saml_finish_acs", kwargs={"organization_slug": "testtenant"}) ) request.user = type("Anonymous", (), {"is_authenticated": False})() + request.session = {} with patch( "allauth.socialaccount.providers.saml.views.get_app_or_404" @@ -6006,6 +6142,7 @@ class TestTenantFinishACSView: reverse("saml_finish_acs", kwargs={"organization_slug": "testtenant"}) ) request.user = users_fixture[0] + request.session = {} with patch( "allauth.socialaccount.providers.saml.views.get_app_or_404" @@ -6047,6 +6184,7 @@ class TestTenantFinishACSView: reverse("saml_finish_acs", kwargs={"organization_slug": "testtenant"}) ) request.user = user + request.session = {} with ( patch( @@ -6113,6 +6251,44 @@ class TestTenantFinishACSView: user.company_name = original_company user.save() + def test_rollback_saml_user_when_error_occurs(self, users_fixture, monkeypatch): + """Test that a user is properly deleted when created during SAML flow and an error occurs""" + monkeypatch.setenv("AUTH_URL", "http://localhost") + + # Create a test user to simulate one created during SAML flow + test_user = User.objects.using(MainRouter.admin_db).create( + email="testuser@example.com", name="Test User" + ) + + request = RequestFactory().get( + reverse("saml_finish_acs", kwargs={"organization_slug": "testtenant"}) + ) + request.user = users_fixture[0] + request.session = {"saml_user_created": test_user.id} + + # Force an exception to trigger rollback + with patch( + "allauth.socialaccount.providers.saml.views.get_app_or_404" + ) as mock_get_app: + mock_get_app.side_effect = Exception("Test error") + + view = TenantFinishACSView.as_view() + response = view(request, organization_slug="testtenant") + + # Verify the user was deleted + assert ( + not User.objects.using(MainRouter.admin_db) + .filter(id=test_user.id) + .exists() + ) + + # Verify session was cleaned up + assert "saml_user_created" not in request.session + + # Verify proper redirect + assert response.status_code == 302 + assert "sso_saml_failed=true" in response.url + @pytest.mark.django_db class TestLighthouseConfigViewSet: diff --git a/api/src/backend/api/v1/mixins.py b/api/src/backend/api/v1/mixins.py index fde14a23c5..1cef76a84c 100644 --- a/api/src/backend/api/v1/mixins.py +++ b/api/src/backend/api/v1/mixins.py @@ -24,20 +24,32 @@ class PaginateByPkMixin: request, # noqa: F841 base_queryset, manager, - select_related: list[str] | None = None, - prefetch_related: list[str] | None = None, + select_related: list | None = None, + prefetch_related: list | None = None, ) -> Response: + """ + Paginate a queryset by primary key. + + This method is useful when you want to paginate a queryset that has been + filtered or annotated in a way that would be lost if you used the default + pagination method. + """ pk_list = base_queryset.values_list("id", flat=True) page = self.paginate_queryset(pk_list) if page is None: return Response(self.get_serializer(base_queryset, many=True).data) queryset = manager.filter(id__in=page) + if select_related: queryset = queryset.select_related(*select_related) if prefetch_related: queryset = queryset.prefetch_related(*prefetch_related) + # Optimize tags loading, if applicable + if hasattr(self, "_optimize_tags_loading"): + queryset = self._optimize_tags_loading(queryset) + queryset = sorted(queryset, key=lambda obj: page.index(obj.id)) serialized = self.get_serializer(queryset, many=True).data diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 2c9b044d51..d19a68f61d 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -9,6 +9,7 @@ from drf_spectacular.utils import extend_schema_field from jwt.exceptions import InvalidKeyError from rest_framework.validators import UniqueTogetherValidator from rest_framework_json_api import serializers +from rest_framework_json_api.relations import SerializerMethodResourceRelatedField from rest_framework_json_api.serializers import ValidationError from rest_framework_simplejwt.exceptions import TokenError from rest_framework_simplejwt.serializers import TokenObtainPairSerializer @@ -999,8 +1000,12 @@ class ResourceSerializer(RLSSerializer): tags = serializers.SerializerMethodField() type_ = serializers.CharField(read_only=True) + failed_findings_count = serializers.IntegerField(read_only=True) - findings = serializers.ResourceRelatedField(many=True, read_only=True) + findings = SerializerMethodResourceRelatedField( + many=True, + read_only=True, + ) class Meta: model = Resource @@ -1016,6 +1021,7 @@ class ResourceSerializer(RLSSerializer): "tags", "provider", "findings", + "failed_findings_count", "url", ] extra_kwargs = { @@ -1037,6 +1043,10 @@ class ResourceSerializer(RLSSerializer): } ) def get_tags(self, obj): + # Use prefetched tags if available to avoid N+1 queries + if hasattr(obj, "prefetched_tags"): + return {tag.key: tag.value for tag in obj.prefetched_tags} + # Fallback to the original method if prefetch is not available return obj.get_tags(self.context.get("tenant_id")) def get_fields(self): @@ -1046,6 +1056,13 @@ class ResourceSerializer(RLSSerializer): fields["type"] = type_ return fields + def get_findings(self, obj): + return ( + obj.latest_findings + if hasattr(obj, "latest_findings") + else obj.findings.all() + ) + class ResourceIncludeSerializer(RLSSerializer): """ @@ -1082,6 +1099,10 @@ class ResourceIncludeSerializer(RLSSerializer): } ) def get_tags(self, obj): + # Use prefetched tags if available to avoid N+1 queries + if hasattr(obj, "prefetched_tags"): + return {tag.key: tag.value for tag in obj.prefetched_tags} + # Fallback to the original method if prefetch is not available return obj.get_tags(self.context.get("tenant_id")) def get_fields(self): @@ -1092,6 +1113,17 @@ class ResourceIncludeSerializer(RLSSerializer): return fields +class ResourceMetadataSerializer(serializers.Serializer): + services = serializers.ListField(child=serializers.CharField(), allow_empty=True) + regions = serializers.ListField(child=serializers.CharField(), allow_empty=True) + types = serializers.ListField(child=serializers.CharField(), allow_empty=True) + # Temporarily disabled until we implement tag filtering in the UI + # tags = serializers.JSONField(help_text="Tags are described as key-value pairs.") + + class Meta: + resource_name = "resources-metadata" + + class FindingSerializer(RLSSerializer): """ Serializer for the Finding model. diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 4f7075ff78..a060c75d8f 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -1,4 +1,5 @@ import glob +import logging import os from datetime import datetime, timedelta, timezone from urllib.parse import urljoin @@ -10,6 +11,7 @@ from allauth.socialaccount.providers.google.views import GoogleOAuth2Adapter from allauth.socialaccount.providers.saml.views import FinishACSView, LoginView from botocore.exceptions import ClientError, NoCredentialsError, ParamValidationError from celery.result import AsyncResult +from config.custom_logging import BackendLogger from config.env import env from config.settings.social_login import ( GITHUB_OAUTH_CALLBACK_URL, @@ -20,7 +22,7 @@ from django.conf import settings as django_settings from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.search import SearchQuery from django.db import transaction -from django.db.models import Count, Exists, F, OuterRef, Prefetch, Q, Sum +from django.db.models import Count, F, Prefetch, Q, Sum from django.db.models.functions import Coalesce from django.http import HttpResponse from django.shortcuts import redirect @@ -76,6 +78,7 @@ from api.filters import ( IntegrationFilter, InvitationFilter, LatestFindingFilter, + LatestResourceFilter, MembershipFilter, ProcessorFilter, ProviderFilter, @@ -106,6 +109,7 @@ from api.models import ( Resource, ResourceFindingMapping, ResourceScanSummary, + ResourceTag, Role, RoleProviderGroupRelationship, SAMLConfiguration, @@ -165,6 +169,7 @@ from api.v1.serializers import ( ProviderSecretUpdateSerializer, ProviderSerializer, ProviderUpdateSerializer, + ResourceMetadataSerializer, ResourceSerializer, RoleCreateSerializer, RoleProviderGroupRelationshipSerializer, @@ -190,6 +195,8 @@ from api.v1.serializers import ( UserUpdateSerializer, ) +logger = logging.getLogger(BackendLogger.API) + CACHE_DECORATOR = cache_control( max_age=django_settings.CACHE_MAX_AGE, stale_while_revalidate=django_settings.CACHE_STALE_WHILE_REVALIDATE, @@ -286,7 +293,7 @@ class SchemaView(SpectacularAPIView): def get(self, request, *args, **kwargs): spectacular_settings.TITLE = "Prowler API" - spectacular_settings.VERSION = "1.9.0" + spectacular_settings.VERSION = "1.10.0" spectacular_settings.DESCRIPTION = ( "Prowler API specification.\n\nThis file is auto-generated." ) @@ -559,10 +566,25 @@ class SAMLConfigurationViewSet(BaseRLSViewSet): class TenantFinishACSView(FinishACSView): + def _rollback_saml_user(self, request): + """Helper function to rollback SAML user if it was just created and validation fails""" + saml_user_id = request.session.get("saml_user_created") + if saml_user_id: + User.objects.using(MainRouter.admin_db).filter(id=saml_user_id).delete() + request.session.pop("saml_user_created", None) + def dispatch(self, request, organization_slug): - super().dispatch(request, organization_slug) + try: + super().dispatch(request, organization_slug) + except Exception as e: + logger.error(f"SAML dispatch failed: {e}") + self._rollback_saml_user(request) + callback_url = env.str("AUTH_URL") + return redirect(f"{callback_url}?sso_saml_failed=true") + user = getattr(request, "user", None) if not user or not user.is_authenticated: + self._rollback_saml_user(request) callback_url = env.str("AUTH_URL") return redirect(f"{callback_url}?sso_saml_failed=true") @@ -585,7 +607,9 @@ class TenantFinishACSView(FinishACSView): SocialApp.DoesNotExist, SocialAccount.DoesNotExist, User.DoesNotExist, - ): + ) as e: + logger.error(f"SAML user is not authenticated: {e}") + self._rollback_saml_user(request) callback_url = env.str("AUTH_URL") return redirect(f"{callback_url}?sso_saml_failed=true") @@ -659,6 +683,7 @@ class TenantFinishACSView(FinishACSView): ) callback_url = env.str("SAML_SSO_CALLBACK_URL") redirect_url = f"{callback_url}?id={saml_token.id}" + request.session.pop("saml_user_created", None) return redirect(redirect_url) @@ -1861,6 +1886,14 @@ class TaskViewSet(BaseRLSViewSet): summary="List all resources", description="Retrieve a list of all resources with options for filtering by various criteria. Resources are " "objects that are discovered by Prowler. They can be anything from a single host to a whole VPC.", + parameters=[ + OpenApiParameter( + name="filter[updated_at]", + description="At least one of the variations of the `filter[updated_at]` filter must be provided.", + required=True, + type=OpenApiTypes.DATE, + ) + ], ), retrieve=extend_schema( tags=["Resource"], @@ -1868,15 +1901,43 @@ class TaskViewSet(BaseRLSViewSet): description="Fetch detailed information about a specific resource by their ID. A Resource is an object that " "is discovered by Prowler. It can be anything from a single host to a whole VPC.", ), + metadata=extend_schema( + tags=["Resource"], + summary="Retrieve metadata values from resources", + description="Fetch unique metadata values from a set of resources. This is useful for dynamic filtering.", + parameters=[ + OpenApiParameter( + name="filter[updated_at]", + description="At least one of the variations of the `filter[updated_at]` filter must be provided.", + required=True, + type=OpenApiTypes.DATE, + ) + ], + filters=True, + ), + latest=extend_schema( + tags=["Resource"], + summary="List the latest resources", + description="Retrieve a list of the latest resources from the latest scans for each provider with options for " + "filtering by various criteria.", + filters=True, + ), + metadata_latest=extend_schema( + tags=["Resource"], + summary="Retrieve metadata values from the latest resources", + description="Fetch unique metadata values from a set of resources from the latest scans for each provider. " + "This is useful for dynamic filtering.", + filters=True, + ), ) @method_decorator(CACHE_DECORATOR, name="list") @method_decorator(CACHE_DECORATOR, name="retrieve") -class ResourceViewSet(BaseRLSViewSet): - queryset = Resource.objects.all() +class ResourceViewSet(PaginateByPkMixin, BaseRLSViewSet): + queryset = Resource.all_objects.all() serializer_class = ResourceSerializer http_method_names = ["get"] filterset_class = ResourceFilter - ordering = ["-inserted_at"] + ordering = ["-failed_findings_count", "-updated_at"] ordering_fields = [ "provider_uid", "uid", @@ -1887,6 +1948,14 @@ class ResourceViewSet(BaseRLSViewSet): "inserted_at", "updated_at", ] + prefetch_for_includes = { + "__all__": [], + "provider": [ + Prefetch( + "provider", queryset=Provider.all_objects.select_related("resources") + ) + ], + } # RBAC required permissions (implicit -> MANAGE_PROVIDERS enable unlimited visibility or check the visibility of # the provider through the provider group) required_permissions = [] @@ -1895,41 +1964,257 @@ class ResourceViewSet(BaseRLSViewSet): user_roles = get_role(self.request.user) if user_roles.unlimited_visibility: # User has unlimited visibility, return all scans - queryset = Resource.objects.filter(tenant_id=self.request.tenant_id) + queryset = Resource.all_objects.filter(tenant_id=self.request.tenant_id) else: # User lacks permission, filter providers based on provider groups associated with the role - queryset = Resource.objects.filter( + queryset = Resource.all_objects.filter( tenant_id=self.request.tenant_id, provider__in=get_providers(user_roles) ) search_value = self.request.query_params.get("filter[search]", None) if search_value: - # Django's ORM will build a LEFT JOIN and OUTER JOIN on the "through" table, resulting in duplicates - # The duplicates then require a `distinct` query search_query = SearchQuery( search_value, config="simple", search_type="plain" ) queryset = queryset.filter( - Q(tags__key=search_value) - | Q(tags__value=search_value) - | Q(tags__text_search=search_query) - | Q(tags__key__contains=search_value) - | Q(tags__value__contains=search_value) - | Q(uid=search_value) - | Q(name=search_value) - | Q(region=search_value) - | Q(service=search_value) - | Q(type=search_value) - | Q(text_search=search_query) - | Q(uid__contains=search_value) - | Q(name__contains=search_value) - | Q(region__contains=search_value) - | Q(service__contains=search_value) - | Q(type__contains=search_value) + Q(text_search=search_query) | Q(tags__text_search=search_query) ).distinct() return queryset + def _optimize_tags_loading(self, queryset): + """Optimize tags loading with prefetch_related to avoid N+1 queries""" + # Use prefetch_related to load all tags in a single query + return queryset.prefetch_related( + Prefetch( + "tags", + queryset=ResourceTag.objects.filter( + tenant_id=self.request.tenant_id + ).select_related(), + to_attr="prefetched_tags", + ) + ) + + def get_serializer_class(self): + if self.action in ["metadata", "metadata_latest"]: + return ResourceMetadataSerializer + return super().get_serializer_class() + + def get_filterset_class(self): + if self.action in ["latest", "metadata_latest"]: + return LatestResourceFilter + return ResourceFilter + + def filter_queryset(self, queryset): + # Do not apply filters when retrieving specific resource + if self.action == "retrieve": + return queryset + return super().filter_queryset(queryset) + + def list(self, request, *args, **kwargs): + filtered_queryset = self.filter_queryset(self.get_queryset()) + return self.paginate_by_pk( + request, + filtered_queryset, + manager=Resource.all_objects, + select_related=["provider"], + prefetch_related=["findings"], + ) + + def retrieve(self, request, *args, **kwargs): + queryset = self._optimize_tags_loading(self.get_queryset()) + instance = get_object_or_404(queryset, pk=kwargs.get("pk")) + mapping_ids = list( + ResourceFindingMapping.objects.filter( + resource=instance, tenant_id=request.tenant_id + ).values_list("finding_id", flat=True) + ) + latest_findings = ( + Finding.all_objects.filter(id__in=mapping_ids, tenant_id=request.tenant_id) + .order_by("uid", "-inserted_at") + .distinct("uid") + ) + setattr(instance, "latest_findings", latest_findings) + serializer = self.get_serializer(instance) + return Response(serializer.data, status=status.HTTP_200_OK) + + @action(detail=False, methods=["get"], url_name="latest") + def latest(self, request): + tenant_id = request.tenant_id + filtered_queryset = self.filter_queryset(self.get_queryset()) + + latest_scan_ids = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + .values_list("id", flat=True) + ) + filtered_queryset = filtered_queryset.filter( + tenant_id=tenant_id, provider__scan__in=latest_scan_ids + ) + + return self.paginate_by_pk( + request, + filtered_queryset, + manager=Resource.all_objects, + select_related=["provider"], + prefetch_related=["findings"], + ) + + @action(detail=False, methods=["get"], url_name="metadata") + def metadata(self, request): + # Force filter validation + self.filter_queryset(self.get_queryset()) + + tenant_id = request.tenant_id + query_params = request.query_params + + queryset = ResourceScanSummary.objects.filter(tenant_id=tenant_id) + + if scans := query_params.get("filter[scan__in]") or query_params.get( + "filter[scan]" + ): + queryset = queryset.filter(scan_id__in=scans.split(",")) + else: + exact = query_params.get("filter[inserted_at]") + gte = query_params.get("filter[inserted_at__gte]") + lte = query_params.get("filter[inserted_at__lte]") + + date_filters = {} + if exact: + date = parse_date(exact) + datetime_start = datetime.combine( + date, datetime.min.time(), tzinfo=timezone.utc + ) + datetime_end = datetime_start + timedelta(days=1) + date_filters["scan_id__gte"] = uuid7_start( + datetime_to_uuid7(datetime_start) + ) + date_filters["scan_id__lt"] = uuid7_start( + datetime_to_uuid7(datetime_end) + ) + else: + if gte: + date_start = parse_date(gte) + datetime_start = datetime.combine( + date_start, datetime.min.time(), tzinfo=timezone.utc + ) + date_filters["scan_id__gte"] = uuid7_start( + datetime_to_uuid7(datetime_start) + ) + if lte: + date_end = parse_date(lte) + datetime_end = datetime.combine( + date_end + timedelta(days=1), + datetime.min.time(), + tzinfo=timezone.utc, + ) + date_filters["scan_id__lt"] = uuid7_start( + datetime_to_uuid7(datetime_end) + ) + + if date_filters: + queryset = queryset.filter(**date_filters) + + if service_filter := query_params.get("filter[service]") or query_params.get( + "filter[service__in]" + ): + queryset = queryset.filter(service__in=service_filter.split(",")) + if region_filter := query_params.get("filter[region]") or query_params.get( + "filter[region__in]" + ): + queryset = queryset.filter(region__in=region_filter.split(",")) + if resource_type_filter := query_params.get("filter[type]") or query_params.get( + "filter[type__in]" + ): + queryset = queryset.filter( + resource_type__in=resource_type_filter.split(",") + ) + + services = list( + queryset.values_list("service", flat=True).distinct().order_by("service") + ) + regions = list( + queryset.values_list("region", flat=True).distinct().order_by("region") + ) + resource_types = list( + queryset.values_list("resource_type", flat=True) + .exclude(resource_type__isnull=True) + .exclude(resource_type__exact="") + .distinct() + .order_by("resource_type") + ) + + result = { + "services": services, + "regions": regions, + "types": resource_types, + } + + serializer = self.get_serializer(data=result) + serializer.is_valid(raise_exception=True) + return Response(serializer.data) + + @action( + detail=False, + methods=["get"], + url_name="metadata_latest", + url_path="metadata/latest", + ) + def metadata_latest(self, request): + tenant_id = request.tenant_id + query_params = request.query_params + + latest_scans_queryset = ( + Scan.all_objects.filter(tenant_id=tenant_id, state=StateChoices.COMPLETED) + .order_by("provider_id", "-inserted_at") + .distinct("provider_id") + ) + + queryset = ResourceScanSummary.objects.filter( + tenant_id=tenant_id, + scan_id__in=latest_scans_queryset.values_list("id", flat=True), + ) + + if service_filter := query_params.get("filter[service]") or query_params.get( + "filter[service__in]" + ): + queryset = queryset.filter(service__in=service_filter.split(",")) + if region_filter := query_params.get("filter[region]") or query_params.get( + "filter[region__in]" + ): + queryset = queryset.filter(region__in=region_filter.split(",")) + if resource_type_filter := query_params.get("filter[type]") or query_params.get( + "filter[type__in]" + ): + queryset = queryset.filter( + resource_type__in=resource_type_filter.split(",") + ) + + services = list( + queryset.values_list("service", flat=True).distinct().order_by("service") + ) + regions = list( + queryset.values_list("region", flat=True).distinct().order_by("region") + ) + resource_types = list( + queryset.values_list("resource_type", flat=True) + .exclude(resource_type__isnull=True) + .exclude(resource_type__exact="") + .distinct() + .order_by("resource_type") + ) + + result = { + "services": services, + "regions": regions, + "types": resource_types, + } + + serializer = self.get_serializer(data=result) + serializer.is_valid(raise_exception=True) + return Response(serializer.data) + @extend_schema_view( list=extend_schema( @@ -2048,17 +2333,7 @@ class FindingViewSet(PaginateByPkMixin, BaseRLSViewSet): search_value, config="simple", search_type="plain" ) - resource_match = Resource.all_objects.filter( - text_search=search_query, - id__in=ResourceFindingMapping.objects.filter( - resource_id=OuterRef("pk"), - tenant_id=tenant_id, - ).values("resource_id"), - ) - - queryset = queryset.filter( - Q(text_search=search_query) | Q(Exists(resource_match)) - ) + queryset = queryset.filter(text_search=search_query) return queryset diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 6024fb3474..7f0f063f75 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -29,6 +29,7 @@ from api.models import ( ProviderSecret, Resource, ResourceTag, + ResourceTagMapping, Role, SAMLConfiguration, SAMLDomainIndex, @@ -654,6 +655,7 @@ def findings_fixture(scans_fixture, resources_fixture): check_metadata={ "CheckId": "test_check_id", "Description": "test description apple sauce", + "servicename": "ec2", }, first_seen_at="2024-01-02T00:00:00Z", ) @@ -680,6 +682,7 @@ def findings_fixture(scans_fixture, resources_fixture): check_metadata={ "CheckId": "test_check_id", "Description": "test description orange juice", + "servicename": "s3", }, first_seen_at="2024-01-02T00:00:00Z", muted=True, @@ -1135,6 +1138,69 @@ def latest_scan_finding(authenticated_client, providers_fixture, resources_fixtu return finding +@pytest.fixture(scope="function") +def latest_scan_resource(authenticated_client, providers_fixture): + provider = providers_fixture[0] + tenant_id = str(providers_fixture[0].tenant_id) + scan = Scan.objects.create( + name="latest completed scan for resource", + provider=provider, + trigger=Scan.TriggerChoices.MANUAL, + state=StateChoices.COMPLETED, + tenant_id=tenant_id, + ) + resource = Resource.objects.create( + tenant_id=tenant_id, + provider=provider, + uid="latest_resource_uid", + name="Latest Resource", + region="us-east-1", + service="ec2", + type="instance", + metadata='{"test": "metadata"}', + details='{"test": "details"}', + ) + + resource_tag = ResourceTag.objects.create( + tenant_id=tenant_id, + key="environment", + value="test", + ) + ResourceTagMapping.objects.create( + tenant_id=tenant_id, + resource=resource, + tag=resource_tag, + ) + + finding = Finding.objects.create( + tenant_id=tenant_id, + uid="test_finding_uid_latest", + scan=scan, + delta="new", + status=Status.FAIL, + status_extended="test status extended ", + impact=Severity.critical, + impact_extended="test impact extended", + severity=Severity.critical, + raw_result={ + "status": Status.FAIL, + "impact": Severity.critical, + "severity": Severity.critical, + }, + tags={"test": "latest"}, + check_id="test_check_id_latest", + check_metadata={ + "CheckId": "test_check_id_latest", + "Description": "test description latest", + }, + first_seen_at="2024-01-02T00:00:00Z", + ) + finding.add_resources([resource]) + + backfill_resource_scan_summaries(tenant_id, str(scan.id)) + return resource + + @pytest.fixture def saml_setup(tenants_fixture): tenant_id = tenants_fixture[0].id diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 118d62693c..af07fe1b67 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from celery.utils.log import get_task_logger from config.settings.celery import CELERY_DEADLOCK_ATTEMPTS from django.db import IntegrityError, OperationalError -from django.db.models import Case, Count, IntegerField, Sum, When +from django.db.models import Case, Count, IntegerField, OuterRef, Subquery, Sum, When from tasks.utils import CustomEncoder from api.compliance import ( @@ -376,12 +376,16 @@ def perform_prowler_scan( def aggregate_findings(tenant_id: str, scan_id: str): """ Aggregates findings for a given scan and stores the results in the ScanSummary table. + Also updates the failed_findings_count for each resource based on the latest findings. This function retrieves all findings associated with a given `scan_id` and calculates various metrics such as counts of failed, passed, and muted findings, as well as their deltas (new, changed, unchanged). The results are grouped by `check_id`, `service`, `severity`, and `region`. These aggregated metrics are then stored in the `ScanSummary` table. + Additionally, it updates the failed_findings_count field for each resource based on the most + recent findings for each finding.uid. + Args: tenant_id (str): The ID of the tenant to which the scan belongs. scan_id (str): The ID of the scan for which findings need to be aggregated. @@ -401,6 +405,8 @@ def aggregate_findings(tenant_id: str, scan_id: str): - muted_new: Muted findings with a delta of 'new'. - muted_changed: Muted findings with a delta of 'changed'. """ + _update_resource_failed_findings_count(tenant_id, scan_id) + with rls_transaction(tenant_id): findings = Finding.objects.filter(tenant_id=tenant_id, scan_id=scan_id) @@ -525,6 +531,53 @@ def aggregate_findings(tenant_id: str, scan_id: str): ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000) +def _update_resource_failed_findings_count(tenant_id: str, scan_id: str): + """ + Update the failed_findings_count field for resources based on the latest findings. + + This function calculates the number of failed findings for each resource by: + 1. Getting the latest finding for each finding.uid + 2. Counting failed findings per resource + 3. Updating the failed_findings_count field for each resource + + Args: + tenant_id (str): The ID of the tenant to which the scan belongs. + scan_id (str): The ID of the scan for which to update resource counts. + """ + + with rls_transaction(tenant_id): + scan = Scan.objects.get(pk=scan_id) + provider_id = scan.provider_id + + resources = list( + Resource.all_objects.filter(tenant_id=tenant_id, provider_id=provider_id) + ) + + # For each resource, calculate failed findings count based on latest findings + for resource in resources: + with rls_transaction(tenant_id): + # Get the latest finding for each finding.uid that affects this resource + latest_findings_subquery = ( + Finding.all_objects.filter( + tenant_id=tenant_id, uid=OuterRef("uid"), resources=resource + ) + .order_by("-inserted_at") + .values("id")[:1] + ) + + # Count failed findings from the latest findings + failed_count = Finding.all_objects.filter( + tenant_id=tenant_id, + resources=resource, + id__in=Subquery(latest_findings_subquery), + status=FindingStatus.FAIL, + muted=False, + ).count() + + resource.failed_findings_count = failed_count + resource.save(update_fields=["failed_findings_count"]) + + def create_compliance_requirements(tenant_id: str, scan_id: str): """ Create detailed compliance requirement overview records for a scan. diff --git a/api/src/backend/tasks/tests/test_scan.py b/api/src/backend/tasks/tests/test_scan.py index 474543a82b..5e6a5d15ef 100644 --- a/api/src/backend/tasks/tests/test_scan.py +++ b/api/src/backend/tasks/tests/test_scan.py @@ -7,6 +7,7 @@ import pytest from tasks.jobs.scan import ( _create_finding_delta, _store_resources, + _update_resource_failed_findings_count, create_compliance_requirements, perform_prowler_scan, ) @@ -1166,3 +1167,85 @@ class TestCreateComplianceRequirements: assert len(req_2_objects) == 2 assert all(obj.requirement_status == "PASS" for obj in req_1_objects) assert all(obj.requirement_status == "FAIL" for obj in req_2_objects) + + +@pytest.mark.django_db +class TestUpdateResourceFailedFindingsCount: + @patch("api.models.Resource.all_objects.filter") + @patch("api.models.Finding.all_objects.filter") + def test_failed_findings_count_update( + self, + mock_finding_filter, + mock_resource_filter, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + + scan.provider = provider + scan.save() + + tenant_id = str(tenant.id) + scan_id = str(scan.id) + + resource1 = MagicMock() + resource1.uid = "res-1" + resource1.failed_findings_count = None + resource1.save = MagicMock() + + resource2 = MagicMock() + resource2.uid = "res-2" + resource2.failed_findings_count = None + resource2.save = MagicMock() + + mock_resource_filter.return_value = [resource1, resource2] + + fake_subquery_qs = MagicMock() + fake_subquery_qs.order_by.return_value = fake_subquery_qs + fake_subquery_qs.values.return_value = fake_subquery_qs + fake_subquery_qs.__getitem__.return_value = fake_subquery_qs + + def finding_filter_side_effect(*args, **kwargs): + if "status" in kwargs: + qs_count = MagicMock() + if kwargs.get("resources") == resource1: + qs_count.count.return_value = 3 + else: + qs_count.count.return_value = 0 + return qs_count + return fake_subquery_qs + + mock_finding_filter.side_effect = finding_filter_side_effect + + _update_resource_failed_findings_count(tenant_id, scan_id) + + # resource1 should have been updated to 3 + assert resource1.failed_findings_count == 3 + resource1.save.assert_called_once_with(update_fields=["failed_findings_count"]) + + # resource2 should have been updated to 0 + assert resource2.failed_findings_count == 0 + resource2.save.assert_called_once_with(update_fields=["failed_findings_count"]) + + @patch("api.models.Resource.all_objects.filter", return_value=[]) + @patch("api.models.Finding.all_objects.filter") + def test_no_resources_no_error( + self, + mock_finding_filter, + mock_resource_filter, + tenants_fixture, + scans_fixture, + providers_fixture, + ): + tenant = tenants_fixture[0] + scan = scans_fixture[0] + provider = providers_fixture[0] + scan.provider = provider + scan.save() + + _update_resource_failed_findings_count(str(tenant.id), str(scan.id)) + + mock_finding_filter.assert_not_called() diff --git a/docs/tutorials/github/authentication.md b/docs/tutorials/github/authentication.md index 5b589921ec..c3a29ad0e8 100644 --- a/docs/tutorials/github/authentication.md +++ b/docs/tutorials/github/authentication.md @@ -27,10 +27,10 @@ prowler github --oauth-app-token oauth_token ``` ### GitHub App Credentials -Use GitHub App credentials by specifying the App ID and the private key. +Use GitHub App credentials by specifying the App ID and the private key path. ```console -prowler github --github-app-id app_id --github-app-key app_key +prowler github --github-app-id app_id --github-app-key-path app_key_path ``` ### Automatic Login Method Detection @@ -38,7 +38,7 @@ If no login method is explicitly provided, Prowler will automatically attempt to 1. `GITHUB_PERSONAL_ACCESS_TOKEN` 2. `GITHUB_OAUTH_APP_TOKEN` -3. `GITHUB_APP_ID` and `GITHUB_APP_KEY` +3. `GITHUB_APP_ID` and `GITHUB_APP_KEY` (where the key is the content of the private key file) ???+ note Ensure the corresponding environment variables are set up before running Prowler for automatic detection if you don't plan to specify the login method. diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index a031d6ff5f..b68c1d1eb4 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to the **Prowler SDK** are documented in this file. - `vm_linux_enforce_ssh_authentication` check for Azure provider [(#8149)](https://github.com/prowler-cloud/prowler/pull/8149) - `vm_ensure_using_approved_images` check for Azure provider [(#8168)](https://github.com/prowler-cloud/prowler/pull/8168) - `vm_scaleset_associated_load_balancer` check for Azure provider [(#8181)](https://github.com/prowler-cloud/prowler/pull/8181) +- Add `test_connection` method to GitHub provider [(#8248)](https://github.com/prowler-cloud/prowler/pull/8248) ### Changed diff --git a/prowler/providers/aws/aws_regions_by_service.json b/prowler/providers/aws/aws_regions_by_service.json index a56854b214..c2e1068fd1 100644 --- a/prowler/providers/aws/aws_regions_by_service.json +++ b/prowler/providers/aws/aws_regions_by_service.json @@ -4024,6 +4024,7 @@ "eu-west-2", "eu-west-3", "il-central-1", + "me-central-1", "me-south-1", "sa-east-1", "us-east-1", @@ -4761,6 +4762,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -4807,6 +4809,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -4853,6 +4856,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -4899,6 +4903,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -4944,6 +4949,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -6145,6 +6151,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -8245,6 +8252,7 @@ "eu-central-1", "eu-north-1", "eu-west-1", + "eu-west-2", "us-east-1", "us-east-2", "us-west-2" @@ -9683,6 +9691,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -9909,6 +9918,7 @@ "aws": [ "af-south-1", "ap-east-1", + "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", @@ -9918,6 +9928,8 @@ "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-7", "ca-central-1", "ca-west-1", "eu-central-1", @@ -9931,6 +9943,7 @@ "il-central-1", "me-central-1", "me-south-1", + "mx-central-1", "sa-east-1", "us-east-1", "us-east-2", @@ -12125,6 +12138,15 @@ ] } }, + "workspaces-instances": { + "regions": { + "aws": [ + "ap-northeast-2" + ], + "aws-cn": [], + "aws-us-gov": [] + } + }, "workspaces-web": { "regions": { "aws": [ diff --git a/prowler/providers/github/exceptions/exceptions.py b/prowler/providers/github/exceptions/exceptions.py index f7058d96ea..b49cce8ebe 100644 --- a/prowler/providers/github/exceptions/exceptions.py +++ b/prowler/providers/github/exceptions/exceptions.py @@ -30,6 +30,10 @@ class GithubBaseException(ProwlerException): "message": "Github invalid App Key or App ID for GitHub APP login", "remediation": "Check user and password and ensure they are properly set up as in your Github account.", }, + (5006, "GithubInvalidProviderIdError"): { + "message": "The provided provider ID does not match with the authenticated user or accessible organizations", + "remediation": "Check the provider ID and ensure it matches the authenticated user or an organization you have access to.", + }, } def __init__(self, code, file=None, original_exception=None, message=None): @@ -93,3 +97,10 @@ class GithubInvalidCredentialsError(GithubCredentialsError): super().__init__( 5005, file=file, original_exception=original_exception, message=message ) + + +class GithubInvalidProviderIdError(GithubCredentialsError): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 5006, file=file, original_exception=original_exception, message=message + ) diff --git a/prowler/providers/github/github_provider.py b/prowler/providers/github/github_provider.py index f217066929..df6c8ee046 100644 --- a/prowler/providers/github/github_provider.py +++ b/prowler/providers/github/github_provider.py @@ -14,11 +14,12 @@ from prowler.config.config import ( from prowler.lib.logger import logger from prowler.lib.mutelist.mutelist import Mutelist from prowler.lib.utils.utils import print_boxes -from prowler.providers.common.models import Audit_Metadata +from prowler.providers.common.models import Audit_Metadata, Connection from prowler.providers.common.provider import Provider from prowler.providers.github.exceptions.exceptions import ( GithubEnvironmentVariableError, GithubInvalidCredentialsError, + GithubInvalidProviderIdError, GithubInvalidTokenError, GithubSetUpIdentityError, GithubSetUpSessionError, @@ -122,14 +123,28 @@ class GithubProvider(Provider): """ logger.info("Instantiating GitHub Provider...") - self._session = self.setup_session( + self._session = GithubProvider.setup_session( personal_access_token, oauth_app_token, github_app_id, github_app_key, ) - self._identity = self.setup_identity() + # Set the authentication method + if personal_access_token: + self._auth_method = "Personal Access Token" + elif oauth_app_token: + self._auth_method = "OAuth App Token" + elif github_app_id and github_app_key: + self._auth_method = "GitHub App Token" + elif environ.get("GITHUB_PERSONAL_ACCESS_TOKEN", ""): + self._auth_method = "Environment Variable for Personal Access Token" + elif environ.get("GITHUB_OAUTH_APP_TOKEN", ""): + self._auth_method = "Environment Variable for OAuth App Token" + elif environ.get("GITHUB_APP_ID", "") and environ.get("GITHUB_APP_KEY", ""): + self._auth_method = "Environment Variables for GitHub App Key and ID" + + self._identity = GithubProvider.setup_identity(self._session) # Audit Config if config_content: @@ -195,12 +210,13 @@ class GithubProvider(Provider): """ return self._mutelist + @staticmethod def setup_session( - self, personal_access_token: str = None, oauth_app_token: str = None, github_app_id: int = 0, github_app_key: str = None, + github_app_key_content: str = None, ) -> GithubSession: """ Returns the GitHub headers responsible authenticating API calls. @@ -210,7 +226,7 @@ class GithubProvider(Provider): oauth_app_token (str): GitHub OAuth App token. github_app_id (int): GitHub App ID. github_app_key (str): GitHub App key. - + github_app_key_content (str): GitHub App key content. Returns: GithubSession: Authenticated session token for API requests. """ @@ -223,18 +239,17 @@ class GithubProvider(Provider): # Ensure that at least one authentication method is selected. Default to environment variable for PAT if none is provided. if personal_access_token: session_token = personal_access_token - self._auth_method = "Personal Access Token" elif oauth_app_token: session_token = oauth_app_token - self._auth_method = "OAuth App Token" - elif github_app_id and github_app_key: + elif github_app_id and (github_app_key or github_app_key_content): app_id = github_app_id - with open(github_app_key, "r") as rsa_key: - app_key = rsa_key.read() - - self._auth_method = "GitHub App Token" + if github_app_key: + with open(github_app_key, "r") as rsa_key: + app_key = rsa_key.read() + else: + app_key = format_rsa_key(github_app_key_content) else: # PAT @@ -242,8 +257,6 @@ class GithubProvider(Provider): "Looking for GITHUB_PERSONAL_ACCESS_TOKEN environment variable as user has not provided any token...." ) session_token = environ.get("GITHUB_PERSONAL_ACCESS_TOKEN", "") - if session_token: - self._auth_method = "Environment Variable for Personal Access Token" if not session_token: # OAUTH @@ -251,8 +264,6 @@ class GithubProvider(Provider): "Looking for GITHUB_OAUTH_APP_TOKEN environment variable as user has not provided any token...." ) session_token = environ.get("GITHUB_OAUTH_APP_TOKEN", "") - if session_token: - self._auth_method = "Environment Variable for OAuth App Token" if not session_token: # APP @@ -260,14 +271,12 @@ class GithubProvider(Provider): "Looking for GITHUB_APP_ID and GITHUB_APP_KEY environment variables as user has not provided any token...." ) app_id = environ.get("GITHUB_APP_ID", "") - app_key = format_rsa_key(environ.get(r"GITHUB_APP_KEY", "")) + app_key = format_rsa_key(environ.get("GITHUB_APP_KEY", "")) if app_id and app_key: - self._auth_method = ( - "Environment Variables for GitHub App Key and ID" - ) + pass - if not self._auth_method: + if not session_token and not (app_id and app_key): raise GithubEnvironmentVariableError( file=os.path.basename(__file__), message="No authentication method selected and not environment variables were found.", @@ -289,8 +298,9 @@ class GithubProvider(Provider): original_exception=error, ) + @staticmethod def setup_identity( - self, + session: GithubSession, ) -> Union[GithubIdentityInfo, GithubAppIdentityInfo]: """ Returns the GitHub identity information @@ -298,12 +308,11 @@ class GithubProvider(Provider): Returns: GithubIdentityInfo | GithubAppIdentityInfo: An instance of GithubIdentityInfo or GithubAppIdentityInfo containing the identity information. """ - credentials = self.session try: retry_config = GithubRetry(total=3) - if credentials.token: - auth = Auth.Token(credentials.token) + if session.token: + auth = Auth.Token(session.token) g = Github(auth=auth, retry=retry_config) try: identity = GithubIdentityInfo( @@ -318,8 +327,8 @@ class GithubProvider(Provider): original_exception=error, ) - elif credentials.id != 0 and credentials.key: - auth = Auth.AppAuth(credentials.id, credentials.key) + elif session.id != 0 and session.key: + auth = Auth.AppAuth(session.id, session.key) gi = GithubIntegration(auth=auth, retry=retry_config) try: identity = GithubAppIdentityInfo(app_id=gi.get_app().id) @@ -360,3 +369,160 @@ class GithubProvider(Provider): f"{Style.BRIGHT}Using the GitHub credentials below:{Style.RESET_ALL}" ) print_boxes(report_lines, report_title) + + @staticmethod + def validate_provider_id( + session: GithubSession, + provider_id: str, + ) -> None: + """ + Validate that the provider ID (username or organization) is accessible with the given credentials. + + Args: + session (GithubSession): The GitHub session with authentication. + provider_id (str): The provider ID to validate (username or organization name). + + Raises: + GithubInvalidProviderIdError: If the provider ID is not accessible with the given credentials. + + Examples: + >>> GithubProvider.validate_provider_id(session, "my-username") + >>> GithubProvider.validate_provider_id(session, "my-organization") + """ + try: + retry_config = GithubRetry(total=3) + + if session.token: + # For Personal Access Token and OAuth App Token + auth = Auth.Token(session.token) + g = Github(auth=auth, retry=retry_config) + + # First check if the provider ID is the authenticated user + authenticated_user = g.get_user() + if authenticated_user.login == provider_id: + return + + # Then check if the provider ID is an organization the token has access to + try: + g.get_organization(provider_id) + return + except Exception: + # Organization doesn't exist or the token doesn't have access to it + pass + + raise GithubInvalidProviderIdError( + file=os.path.basename(__file__), + message=f"The provider ID '{provider_id}' is not accessible with the provided credentials. " + f"Authenticated user: {authenticated_user.login}", + ) + + elif session.id != 0 and session.key: + # For GitHub App + auth = Auth.AppAuth(session.id, session.key) + gi = GithubIntegration(auth=auth, retry=retry_config) + + # Check if the provider ID is in the app's installations + for installation in gi.get_installations(): + try: + # Check if the installation id is the username or organization id + account_login = installation.raw_data.get("account", {}).get( + "login" + ) + if account_login == provider_id: + return + except Exception: + continue + + raise GithubInvalidProviderIdError( + file=os.path.basename(__file__), + message=f"The provider ID '{provider_id}' is not accessible with the provided GitHub App credentials.", + ) + + except GithubInvalidProviderIdError: + # Re-raise the specific exception + raise + except Exception as error: + logger.critical( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + raise GithubInvalidProviderIdError( + file=os.path.basename(__file__), + original_exception=error, + message=f"Error validating provider ID '{provider_id}'", + ) + + @staticmethod + def test_connection( + personal_access_token: str = "", + oauth_app_token: str = "", + github_app_key: str = "", + github_app_key_content: str = "", + github_app_id: int = 0, + raise_on_exception: bool = True, + provider_id: str = None, + ) -> Connection: + """Test connection to GitHub. + + Test the connection to GitHub using the provided credentials. + + Args: + personal_access_token (str): GitHub personal access token. + oauth_app_token (str): GitHub OAuth App token. + github_app_key (str): GitHub App key. + github_app_key_content (str): GitHub App key content. + github_app_id (int): GitHub App ID. + raise_on_exception (bool): Flag indicating whether to raise an exception if the connection fails. + provider_id (str): The provider ID, in this case it's the GitHub organization/username. + + Returns: + Connection: Connection object with success status or error information. + + Raises: + Exception: If failed to test the connection to GitHub. + GithubEnvironmentVariableError: If environment variables are missing. + GithubInvalidTokenError: If the provided token is invalid. + GithubInvalidCredentialsError: If the provided App credentials are invalid. + GithubSetUpSessionError: If there is an error setting up the session. + GithubSetUpIdentityError: If there is an error setting up the identity. + GithubInvalidProviderIdError: If the provided provider ID is not accessible with the given credentials. + + Examples: + >>> GithubProvider.test_connection(personal_access_token="ghp_xxxxxxxxxxxxxxxx") + Connection(is_connected=True) + >>> GithubProvider.test_connection(github_app_id=12345, github_app_key="/path/to/key.pem") + Connection(is_connected=True) + >>> GithubProvider.test_connection(provider_id="my-org") + Connection(is_connected=True) + """ + try: + # Set up the GitHub session + session = GithubProvider.setup_session( + personal_access_token=personal_access_token, + oauth_app_token=oauth_app_token, + github_app_id=github_app_id, + github_app_key=github_app_key, + github_app_key_content=github_app_key_content, + ) + + # Set up the identity to test the connection + GithubProvider.setup_identity(session) + + # Validate provider ID if provided + if provider_id: + GithubProvider.validate_provider_id(session, provider_id) + + return Connection(is_connected=True) + except GithubInvalidProviderIdError as provider_id_error: + logger.critical( + f"{provider_id_error.__class__.__name__}[{provider_id_error.__traceback__.tb_lineno}]: {provider_id_error}" + ) + if raise_on_exception: + raise provider_id_error + return Connection(error=provider_id_error) + except Exception as error: + logger.critical( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + if raise_on_exception: + raise error + return Connection(error=error) diff --git a/prowler/providers/github/lib/arguments/arguments.py b/prowler/providers/github/lib/arguments/arguments.py index 530be5b012..9ba2415cdd 100644 --- a/prowler/providers/github/lib/arguments/arguments.py +++ b/prowler/providers/github/lib/arguments/arguments.py @@ -31,6 +31,7 @@ def init_parser(self): ) github_auth_subparser.add_argument( "--github-app-key", + "--github-app-key-path", nargs="?", help="GitHub App Key Path to log in against GitHub", default=None, diff --git a/tests/providers/github/github_provider_test.py b/tests/providers/github/github_provider_test.py index 60b705b629..88ed4dcfdb 100644 --- a/tests/providers/github/github_provider_test.py +++ b/tests/providers/github/github_provider_test.py @@ -1,9 +1,20 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch + +import pytest from prowler.config.config import ( default_fixer_config_file_path, load_and_validate_config_file, ) +from prowler.providers.common.models import Connection +from prowler.providers.github.exceptions.exceptions import ( + GithubEnvironmentVariableError, + GithubInvalidCredentialsError, + GithubInvalidProviderIdError, + GithubInvalidTokenError, + GithubSetUpIdentityError, + GithubSetUpSessionError, +) from prowler.providers.github.github_provider import GithubProvider from prowler.providers.github.models import ( GithubAppIdentityInfo, @@ -141,3 +152,492 @@ class TestGitHubProvider: "inactive_not_archived_days_threshold": 180, } assert provider._fixer_config == fixer_config + + def test_test_connection_with_personal_access_token_success(self): + """Test successful connection with personal access token.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + ): + connection = GithubProvider.test_connection(personal_access_token=PAT_TOKEN) + + assert isinstance(connection, Connection) + assert connection.is_connected is True + assert connection.error is None + + def test_test_connection_with_oauth_app_token_success(self): + """Test successful connection with OAuth app token.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=OAUTH_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + ): + connection = GithubProvider.test_connection(oauth_app_token=OAUTH_TOKEN) + + assert isinstance(connection, Connection) + assert connection.is_connected is True + assert connection.error is None + + def test_test_connection_with_github_app_success(self): + """Test successful connection with GitHub App credentials.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token="", id=APP_ID, key=APP_KEY), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubAppIdentityInfo(app_id=APP_ID), + ), + ): + connection = GithubProvider.test_connection( + github_app_id=APP_ID, github_app_key=APP_KEY + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is True + assert connection.error is None + + def test_test_connection_with_invalid_token_raises_exception(self): + """Test connection with invalid token raises exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token="invalid-token", id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + side_effect=GithubInvalidTokenError( + original_exception=Exception("Invalid token") + ), + ), + ): + with pytest.raises(GithubInvalidTokenError): + GithubProvider.test_connection(personal_access_token="invalid-token") + + def test_test_connection_with_invalid_token_no_raise(self): + """Test connection with invalid token without raising exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token="invalid-token", id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + side_effect=GithubInvalidTokenError( + original_exception=Exception("Invalid token") + ), + ), + ): + connection = GithubProvider.test_connection( + personal_access_token="invalid-token", raise_on_exception=False + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, GithubInvalidTokenError) + + def test_test_connection_with_invalid_app_credentials_raises_exception(self): + """Test connection with invalid GitHub App credentials raises exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token="", id=APP_ID, key="invalid-key"), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + side_effect=GithubInvalidCredentialsError( + original_exception=Exception("Invalid credentials") + ), + ), + ): + with pytest.raises(GithubInvalidCredentialsError): + GithubProvider.test_connection( + github_app_id=APP_ID, github_app_key="invalid-key" + ) + + def test_test_connection_with_invalid_app_credentials_no_raise(self): + """Test connection with invalid GitHub App credentials without raising exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token="", id=APP_ID, key="invalid-key"), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + side_effect=GithubInvalidCredentialsError( + original_exception=Exception("Invalid credentials") + ), + ), + ): + connection = GithubProvider.test_connection( + github_app_id=APP_ID, + github_app_key="invalid-key", + raise_on_exception=False, + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, GithubInvalidCredentialsError) + + def test_test_connection_setup_session_error_raises_exception(self): + """Test connection when setup_session raises an exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + side_effect=GithubSetUpSessionError( + original_exception=Exception("Setup error") + ), + ), + patch("prowler.providers.github.github_provider.logger") as mock_logger, + ): + with pytest.raises(GithubSetUpSessionError): + GithubProvider.test_connection(personal_access_token=PAT_TOKEN) + + mock_logger.critical.assert_called_once() + + def test_test_connection_setup_session_error_no_raise(self): + """Test connection when setup_session raises an exception without raising.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + side_effect=GithubSetUpSessionError( + original_exception=Exception("Setup error") + ), + ), + patch("prowler.providers.github.github_provider.logger") as mock_logger, + ): + connection = GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, raise_on_exception=False + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, GithubSetUpSessionError) + mock_logger.critical.assert_called_once() + + def test_test_connection_environment_variable_error_raises_exception(self): + """Test connection when environment variable error occurs.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + side_effect=GithubEnvironmentVariableError( + file="test_file.py", message="Env error" + ), + ), + patch("prowler.providers.github.github_provider.logger") as mock_logger, + ): + with pytest.raises(GithubEnvironmentVariableError): + GithubProvider.test_connection(personal_access_token=PAT_TOKEN) + + mock_logger.critical.assert_called_once() + + def test_test_connection_environment_variable_error_no_raise(self): + """Test connection when environment variable error occurs without raising.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + side_effect=GithubEnvironmentVariableError( + file="test_file.py", message="Env error" + ), + ), + patch("prowler.providers.github.github_provider.logger") as mock_logger, + ): + connection = GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, raise_on_exception=False + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, GithubEnvironmentVariableError) + mock_logger.critical.assert_called_once() + + def test_test_connection_setup_identity_error_raises_exception(self): + """Test connection when setup_identity raises an exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + side_effect=GithubSetUpIdentityError( + original_exception=Exception("Identity error") + ), + ), + ): + with pytest.raises(GithubSetUpIdentityError): + GithubProvider.test_connection(personal_access_token=PAT_TOKEN) + + def test_test_connection_setup_identity_error_no_raise(self): + """Test connection when setup_identity raises an exception without raising.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + side_effect=GithubSetUpIdentityError( + original_exception=Exception("Identity error") + ), + ), + ): + connection = GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, raise_on_exception=False + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, GithubSetUpIdentityError) + + def test_test_connection_generic_exception_raises_exception(self): + """Test connection when a generic exception occurs.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + side_effect=Exception("Generic error"), + ), + patch("prowler.providers.github.github_provider.logger") as mock_logger, + ): + with pytest.raises(Exception) as exc_info: + GithubProvider.test_connection(personal_access_token=PAT_TOKEN) + + assert str(exc_info.value) == "Generic error" + mock_logger.critical.assert_called_once() + + def test_test_connection_generic_exception_no_raise(self): + """Test connection when a generic exception occurs without raising.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + side_effect=Exception("Generic error"), + ), + patch("prowler.providers.github.github_provider.logger") as mock_logger, + ): + connection = GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, raise_on_exception=False + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, Exception) + assert str(connection.error) == "Generic error" + mock_logger.critical.assert_called_once() + + def test_test_connection_with_provider_id(self): + """Test connection with provider_id parameter (should validate provider ID).""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.validate_provider_id", + return_value=None, + ), + ): + connection = GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, provider_id="test-org" + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is True + assert connection.error is None + + def test_test_connection_with_invalid_provider_id_raises_exception(self): + """Test connection with invalid provider_id raises exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.validate_provider_id", + side_effect=GithubInvalidProviderIdError( + file="test_file.py", message="Invalid provider ID" + ), + ), + ): + with pytest.raises(GithubInvalidProviderIdError): + GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, provider_id="invalid-org" + ) + + def test_test_connection_with_invalid_provider_id_no_raise(self): + """Test connection with invalid provider_id without raising exception.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.validate_provider_id", + side_effect=GithubInvalidProviderIdError( + file="test_file.py", message="Invalid provider ID" + ), + ), + ): + connection = GithubProvider.test_connection( + personal_access_token=PAT_TOKEN, + provider_id="invalid-org", + raise_on_exception=False, + ) + + assert isinstance(connection, Connection) + assert connection.is_connected is False + assert isinstance(connection.error, GithubInvalidProviderIdError) + + def test_validate_provider_id_with_valid_user(self): + """Test validate_provider_id with valid user (matches authenticated user).""" + mock_session = GithubSession(token=PAT_TOKEN, id="", key="") + + with ( + patch("prowler.providers.github.github_provider.Auth.Token"), + patch("prowler.providers.github.github_provider.Github") as mock_github, + patch("prowler.providers.github.github_provider.GithubRetry"), + ): + # Mock the GitHub client and user + mock_user = MagicMock() + mock_user.login = "test-user" + mock_github_instance = MagicMock() + mock_github_instance.get_user.return_value = mock_user + mock_github.return_value = mock_github_instance + + # Should not raise an exception + GithubProvider.validate_provider_id(mock_session, "test-user") + + def test_validate_provider_id_with_valid_organization(self): + """Test validate_provider_id with valid organization.""" + mock_session = GithubSession(token=PAT_TOKEN, id="", key="") + + with ( + patch("prowler.providers.github.github_provider.Auth.Token"), + patch("prowler.providers.github.github_provider.Github") as mock_github, + patch("prowler.providers.github.github_provider.GithubRetry"), + ): + # Mock the GitHub client and user + mock_user = MagicMock() + mock_user.login = "test-user" + mock_github_instance = MagicMock() + mock_github_instance.get_user.return_value = mock_user + mock_github_instance.get_organization.return_value = ( + MagicMock() + ) # Organization exists + mock_github.return_value = mock_github_instance + + # Should not raise an exception + GithubProvider.validate_provider_id(mock_session, "test-org") + + def test_validate_provider_id_with_invalid_provider_id(self): + """Test validate_provider_id with invalid provider ID.""" + mock_session = GithubSession(token=PAT_TOKEN, id="", key="") + + with ( + patch("prowler.providers.github.github_provider.Auth.Token"), + patch("prowler.providers.github.github_provider.Github") as mock_github, + patch("prowler.providers.github.github_provider.GithubRetry"), + ): + # Mock the GitHub client and user + mock_user = MagicMock() + mock_user.login = "test-user" + mock_github_instance = MagicMock() + mock_github_instance.get_user.return_value = mock_user + mock_github_instance.get_organization.side_effect = Exception("Not found") + mock_github_instance.get_user.side_effect = [ + mock_user, + Exception("Not found"), + ] + mock_github.return_value = mock_github_instance + + with pytest.raises(GithubInvalidProviderIdError): + GithubProvider.validate_provider_id(mock_session, "invalid-provider") + + def test_validate_provider_id_with_github_app(self): + """Test validate_provider_id with GitHub App credentials.""" + mock_session = GithubSession(token="", id=APP_ID, key=APP_KEY) + + with ( + patch("prowler.providers.github.github_provider.Auth.AppAuth"), + patch( + "prowler.providers.github.github_provider.GithubIntegration" + ) as mock_integration, + patch("prowler.providers.github.github_provider.GithubRetry"), + ): + # Mock the GitHub integration and installations + mock_installation = MagicMock() + mock_installation.raw_data = {"account": {"login": "test-org"}} + mock_integration_instance = MagicMock() + mock_integration_instance.get_installations.return_value = [ + mock_installation + ] + mock_integration.return_value = mock_integration_instance + + # Should not raise an exception + GithubProvider.validate_provider_id(mock_session, "test-org") + + def test_validate_provider_id_with_github_app_invalid_org(self): + """Test validate_provider_id with GitHub App credentials and invalid organization.""" + mock_session = GithubSession(token="", id=APP_ID, key=APP_KEY) + + with ( + patch("prowler.providers.github.github_provider.Auth.AppAuth"), + patch( + "prowler.providers.github.github_provider.GithubIntegration" + ) as mock_integration, + patch("prowler.providers.github.github_provider.GithubRetry"), + ): + # Mock the GitHub integration and installations + mock_installation = MagicMock() + mock_installation.raw_data = {"account": {"login": "other-org"}} + mock_integration_instance = MagicMock() + mock_integration_instance.get_installations.return_value = [ + mock_installation + ] + mock_integration.return_value = mock_integration_instance + + with pytest.raises(GithubInvalidProviderIdError): + GithubProvider.validate_provider_id(mock_session, "invalid-org") diff --git a/ui/components/auth/oss/auth-form.tsx b/ui/components/auth/oss/auth-form.tsx index 211d238341..a965204dbc 100644 --- a/ui/components/auth/oss/auth-form.tsx +++ b/ui/components/auth/oss/auth-form.tsx @@ -3,7 +3,8 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { Icon } from "@iconify/react"; import { Button, Checkbox, Divider, Link, Tooltip } from "@nextui-org/react"; -import { useRouter } from "next/navigation"; +import { useRouter, useSearchParams } from "next/navigation"; +import { useEffect } from "react"; import { useForm } from "react-hook-form"; import { z } from "zod"; @@ -41,6 +42,24 @@ export const AuthForm = ({ }) => { const formSchema = authFormSchema(type); const router = useRouter(); + const searchParams = useSearchParams(); + const { toast } = useToast(); + + useEffect(() => { + const samlError = searchParams.get("sso_saml_failed"); + + if (samlError) { + // Add a delay to the toast to ensure it is rendered + setTimeout(() => { + toast({ + variant: "destructive", + title: "SAML Authentication Error", + description: + "An error occurred while attempting to login via your Identity Provider (IdP). Please check your IdP configuration.", + }); + }, 100); + } + }, [searchParams, toast]); const form = useForm>({ resolver: zodResolver(formSchema), @@ -58,7 +77,6 @@ export const AuthForm = ({ }); const isLoading = form.formState.isSubmitting; - const { toast } = useToast(); const isSamlMode = form.watch("isSamlMode"); const onSubmit = async (data: z.infer) => {