diff --git a/api/changelog.d/jira-issue-dedup.added.md b/api/changelog.d/jira-issue-dedup.added.md new file mode 100644 index 0000000000..27de308fb0 --- /dev/null +++ b/api/changelog.d/jira-issue-dedup.added.md @@ -0,0 +1 @@ +Jira issues created from findings are now tracked per finding UID in the new `jira_issues` table and exposed through `GET /api/v1/jira-issues`; sending a finding that already has an open Jira issue skips it (reported as `skipped_count`), and findings whose issue was closed or deleted in Jira get a new issue that replaces the link diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index a7f888b453..70103185c2 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -17,6 +17,7 @@ from api.models import ( FindingGroupDailySummary, Integration, Invitation, + JiraIssue, LighthouseProviderConfiguration, LighthouseProviderModels, Membership, @@ -1900,3 +1901,30 @@ class ComplianceWatchlistFilter(BaseProviderFilter): class Meta(BaseProviderFilter.Meta): model = ProviderComplianceScore + + +class JiraIssueFilter(BaseProviderFilter): + finding_uid = CharFilter(field_name="finding_uid", lookup_expr="exact") + finding_uid__in = CharInFilter(field_name="finding_uid", lookup_expr="in") + finding_id = UUIDFilter(field_name="finding_id", lookup_expr="exact") + finding_id__in = UUIDInFilter(field_name="finding_id", lookup_expr="in") + integration = UUIDFilter(field_name="integration__id", lookup_expr="exact") + integration__in = UUIDInFilter(field_name="integration__id", lookup_expr="in") + issue_key = CharFilter(field_name="issue_key", lookup_expr="exact") + issue_key__in = CharInFilter(field_name="issue_key", lookup_expr="in") + issue_status_category = ChoiceFilter( + choices=JiraIssue.StatusCategoryChoices.choices + ) + issue_status_category__in = ChoiceInFilter( + choices=JiraIssue.StatusCategoryChoices.choices, + field_name="issue_status_category", + lookup_expr="in", + ) + + class Meta: + model = JiraIssue + fields = { + "inserted_at": ["date", "gte", "lte"], + "updated_at": ["date", "gte", "lte"], + "project_key": ["exact", "in"], + } diff --git a/api/src/backend/api/migrations/0098_jira_issues.py b/api/src/backend/api/migrations/0098_jira_issues.py new file mode 100644 index 0000000000..99f8478596 --- /dev/null +++ b/api/src/backend/api/migrations/0098_jira_issues.py @@ -0,0 +1,104 @@ +import uuid + +import api.rls +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0097_attack_paths_scan_db_defaults"), + ] + + operations = [ + migrations.CreateModel( + name="JiraIssue", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("finding_uid", models.CharField(max_length=300)), + ("finding_id", models.UUIDField()), + ( + "issue_key", + models.CharField(blank=True, default="", max_length=64), + ), + ( + "issue_id", + models.CharField(blank=True, default="", max_length=64), + ), + ( + "issue_url", + models.URLField(blank=True, default="", max_length=2048), + ), + ("project_key", models.CharField(max_length=64)), + ( + "issue_status", + models.CharField(blank=True, default="", max_length=64), + ), + ( + "issue_status_category", + models.CharField( + blank=True, + choices=[ + ("new", "New"), + ("indeterminate", "In progress"), + ("done", "Done"), + ], + default="", + max_length=16, + ), + ), + ("status_synced_at", models.DateTimeField(blank=True, null=True)), + ( + "integration", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="jira_issues", + to="api.integration", + ), + ), + ( + "provider", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="jira_issues", + to="api.provider", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "jira_issues", + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="jiraissue", + constraint=models.UniqueConstraint( + fields=("tenant_id", "integration_id", "provider_id", "finding_uid"), + name="unique_jira_issue_per_finding", + ), + ), + migrations.AddConstraint( + model_name="jiraissue", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_jiraissue", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + ] diff --git a/api/src/backend/api/migrations/0099_jira_issues_indexes.py b/api/src/backend/api/migrations/0099_jira_issues_indexes.py new file mode 100644 index 0000000000..be8b7ae971 --- /dev/null +++ b/api/src/backend/api/migrations/0099_jira_issues_indexes.py @@ -0,0 +1,17 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0098_jira_issues"), + ] + + operations = [ + migrations.AddIndex( + model_name="jiraissue", + index=models.Index( + fields=["tenant_id", "provider_id", "finding_uid"], + name="ji_tenant_prov_uid_idx", + ), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index a280708d53..ce28b8f916 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -3113,3 +3113,77 @@ class TenantComplianceSummary(RowLevelSecurityProtectedModel): statements=["SELECT", "INSERT", "UPDATE", "DELETE"], ), ] + + +class JiraIssue(RowLevelSecurityProtectedModel): + """Jira issue created from a finding through a Jira integration. + + One row per (integration, provider, finding uid). Keyed on the finding ``uid`` + rather than the per-scan finding id so the link survives rescans, which is + what lets a repeated send be recognised as already ticketed. Only the latest + ticket is kept: when a linked issue is closed or deleted in Jira and the + finding is sent again, the row is updated to point at the new issue. + """ + + class StatusCategoryChoices(models.TextChoices): + NEW = "new", _("New") + INDETERMINATE = "indeterminate", _("In progress") + DONE = "done", _("Done") + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + integration = models.ForeignKey( + Integration, on_delete=models.CASCADE, related_name="jira_issues" + ) + provider = models.ForeignKey( + Provider, on_delete=models.CASCADE, related_name="jira_issues" + ) + finding_uid = models.CharField(max_length=300) + # Last finding record that was sent; informational, findings are partitioned + # and rotate per scan so this is not a foreign key + finding_id = models.UUIDField() + # Empty while the issue is being created (reservation), filled after Jira + # confirms the creation + issue_key = models.CharField(max_length=64, blank=True, default="") + issue_id = models.CharField(max_length=64, blank=True, default="") + issue_url = models.URLField(max_length=2048, blank=True, default="") + project_key = models.CharField(max_length=64) + issue_status = models.CharField(max_length=64, blank=True, default="") + issue_status_category = models.CharField( + max_length=16, choices=StatusCategoryChoices.choices, blank=True, default="" + ) + status_synced_at = models.DateTimeField(null=True, blank=True) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "jira_issues" + + constraints = [ + models.UniqueConstraint( + fields=("tenant_id", "integration_id", "provider_id", "finding_uid"), + name="unique_jira_issue_per_finding", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + indexes = [ + models.Index( + fields=["tenant_id", "provider_id", "finding_uid"], + name="ji_tenant_prov_uid_idx", + ), + ] + + class JSONAPIMeta: + resource_name = "jira-issues" + + @property + def is_linked(self) -> bool: + """Whether the row points at a confirmed Jira issue (not a reservation).""" + return bool(self.issue_key) + + @property + def is_done(self) -> bool: + return self.issue_status_category == self.StatusCategoryChoices.DONE diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 6cb1d28392..0333d53d6e 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -7107,6 +7107,405 @@ paths: schema: $ref: '#/components/schemas/OpenApiResponseResponse' description: '' + /api/v1/jira-issues: + get: + operationId: api_v1_jira_issues_list + description: Retrieve the Jira issues created from findings through Jira integrations. + Each entry links a finding UID to the latest Jira issue created for it, with + the last status observed in Jira. Use `filter[finding_uid__in]` and `filter[provider_id]` + to check whether specific findings already have a ticket. + summary: List Jira issues linked to findings + parameters: + - in: query + name: fields[jira-issues] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - finding_uid + - finding_id + - issue_key + - issue_id + - issue_url + - project_key + - issue_status + - issue_status_category + - status_synced_at + - integration + - provider + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[finding_id] + schema: + type: string + format: uuid + - in: query + name: filter[finding_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[finding_uid] + schema: + type: string + - in: query + name: filter[finding_uid__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[inserted_at__date] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[integration] + schema: + type: string + format: uuid + - in: query + name: filter[integration__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[issue_key] + schema: + type: string + - in: query + name: filter[issue_key__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[issue_status_category] + schema: + type: string + x-spec-enum-id: 6e5c623f6bbdd92d + enum: + - done + - indeterminate + - new + description: |- + * `new` - New + * `indeterminate` - In progress + * `done` - Done + - in: query + name: filter[issue_status_category__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 6e5c623f6bbdd92d + enum: + - done + - indeterminate + - new + description: |- + Multiple values may be separated by commas. + + * `new` - New + * `indeterminate` - In progress + * `done` - Done + explode: false + style: form + - in: query + name: filter[project_key] + schema: + type: string + - in: query + name: filter[project_key__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_groups] + schema: + type: string + format: uuid + - in: query + name: filter[provider_groups__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_id__in] + schema: + type: array + items: + type: string + format: uuid + description: Multiple values may be separated by commas. + explode: false + style: form + - in: query + name: filter[provider_type] + schema: + type: string + x-spec-enum-id: 203afc16daac9b64 + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - okta + - openstack + - oraclecloud + - vercel + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + * `okta` - Okta + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + x-spec-enum-id: 203afc16daac9b64 + enum: + - alibabacloud + - aws + - azure + - cloudflare + - gcp + - github + - googleworkspace + - iac + - image + - kubernetes + - m365 + - mongodbatlas + - okta + - openstack + - oraclecloud + - vercel + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + * `m365` - M365 + * `github` - GitHub + * `mongodbatlas` - MongoDB Atlas + * `iac` - IaC + * `oraclecloud` - Oracle Cloud Infrastructure + * `alibabacloud` - Alibaba Cloud + * `cloudflare` - Cloudflare + * `openstack` - OpenStack + * `image` - Image + * `googleworkspace` - Google Workspace + * `vercel` - Vercel + * `okta` - Okta + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: filter[updated_at__date] + schema: + type: string + format: date + - in: query + name: filter[updated_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[updated_at__lte] + schema: + type: string + format: date-time + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + - name: page[number] + required: false + in: query + description: A page number within the paginated result set. + schema: + type: integer + - name: page[size] + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - inserted_at + - -inserted_at + - updated_at + - -updated_at + - issue_key + - -issue_key + - project_key + - -project_key + - issue_status + - -issue_status + - status_synced_at + - -status_synced_at + explode: false + tags: + - Integration + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/PaginatedJiraIssueList' + description: '' + /api/v1/jira-issues/{id}: + get: + operationId: api_v1_jira_issues_retrieve + description: Fetch the Jira issue linked to a finding by the link ID. + summary: Retrieve a Jira issue link + parameters: + - in: query + name: fields[jira-issues] + schema: + type: array + items: + type: string + enum: + - inserted_at + - updated_at + - finding_uid + - finding_id + - issue_key + - issue_id + - issue_url + - project_key + - issue_status + - issue_status_category + - status_synced_at + - integration + - provider + - url + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: path + name: id + schema: + type: string + format: uuid + description: A UUID string identifying this jira issue. + required: true + - in: query + name: include + schema: + type: array + items: + type: string + enum: + - provider + description: include query parameter to allow the client to customize which + related resources should be returned. + explode: false + tags: + - Integration + security: + - JWT or API Key: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/JiraIssueResponse' + description: '' /api/v1/lighthouse-configurations: get: operationId: api_v1_lighthouse_configurations_list @@ -18618,6 +19017,134 @@ components: $ref: '#/components/schemas/InvitationUpdate' required: - data + JiraIssue: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + type: string + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + enum: + - jira-issues + id: + type: string + format: uuid + attributes: + type: object + properties: + inserted_at: + type: string + format: date-time + readOnly: true + updated_at: + type: string + format: date-time + readOnly: true + finding_uid: + type: string + readOnly: true + finding_id: + type: string + format: uuid + readOnly: true + issue_key: + type: string + readOnly: true + issue_id: + type: string + readOnly: true + issue_url: + type: string + format: uri + readOnly: true + project_key: + type: string + readOnly: true + issue_status: + type: string + readOnly: true + issue_status_category: + enum: + - new + - indeterminate + - done + type: string + description: |- + * `new` - New + * `indeterminate` - In progress + * `done` - Done + x-spec-enum-id: 6e5c623f6bbdd92d + readOnly: true + status_synced_at: + type: string + format: date-time + readOnly: true + nullable: true + relationships: + type: object + properties: + integration: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - integrations + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + provider: + type: object + properties: + data: + type: object + properties: + id: + type: string + format: uuid + type: + type: string + enum: + - providers + title: Resource Type Name + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common + attributes and relationships. + required: + - id + - type + required: + - data + description: The identifier of the related object. + title: Resource Identifier + readOnly: true + JiraIssueResponse: + type: object + properties: + data: + $ref: '#/components/schemas/JiraIssue' + required: + - data LighthouseConfig: type: object required: @@ -20227,6 +20754,15 @@ components: $ref: '#/components/schemas/Invitation' required: - data + PaginatedJiraIssueList: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/JiraIssue' + required: + - data PaginatedLighthouseConfigList: type: object properties: diff --git a/api/src/backend/api/tests/test_models.py b/api/src/backend/api/tests/test_models.py index 5095da3a0e..0d7db6918c 100644 --- a/api/src/backend/api/tests/test_models.py +++ b/api/src/backend/api/tests/test_models.py @@ -4,6 +4,7 @@ import pytest from allauth.socialaccount.models import SocialApp from api.db_router import MainRouter from api.models import ( + JiraIssue, ProviderComplianceScore, Resource, ResourceTag, @@ -524,3 +525,55 @@ class TestTenantComplianceSummaryModel: assert summary1.id != summary2.id assert summary1.requirements_passed != summary2.requirements_passed + + +@pytest.mark.django_db +class TestJiraIssueModel: + def test_create_jira_issue( + self, jira_integration_fixture, aws_provider, findings_fixture + ): + finding = findings_fixture[0] + issue = JiraIssue.objects.create( + tenant_id=jira_integration_fixture.tenant_id, + integration=jira_integration_fixture, + provider=aws_provider, + finding_uid=finding.uid, + finding_id=finding.id, + issue_key="TEST-1", + project_key="TEST", + ) + assert issue.is_linked + assert not issue.is_done + assert issue.issue_status_category == "" + + def test_reservation_is_not_linked( + self, jira_integration_fixture, aws_provider, findings_fixture + ): + finding = findings_fixture[0] + issue = JiraIssue.objects.create( + tenant_id=jira_integration_fixture.tenant_id, + integration=jira_integration_fixture, + provider=aws_provider, + finding_uid=finding.uid, + finding_id=finding.id, + project_key="TEST", + ) + assert not issue.is_linked + + def test_unique_per_integration_provider_and_finding_uid( + self, jira_integration_fixture, aws_provider_pair, findings_fixture + ): + provider, provider2 = aws_provider_pair + finding = findings_fixture[0] + common = { + "tenant_id": jira_integration_fixture.tenant_id, + "integration": jira_integration_fixture, + "finding_uid": finding.uid, + "finding_id": finding.id, + "project_key": "TEST", + } + JiraIssue.objects.create(provider=provider, issue_key="TEST-1", **common) + # Same finding uid on another provider is a different finding + JiraIssue.objects.create(provider=provider2, issue_key="TEST-2", **common) + with pytest.raises(IntegrityError): + JiraIssue.objects.create(provider=provider, issue_key="TEST-3", **common) diff --git a/api/src/backend/api/tests/test_rbac.py b/api/src/backend/api/tests/test_rbac.py index 772742eeda..e7785bc473 100644 --- a/api/src/backend/api/tests/test_rbac.py +++ b/api/src/backend/api/tests/test_rbac.py @@ -1591,6 +1591,19 @@ class TestLimitedVisibility: assert response.status_code == status.HTTP_204_NO_CONTENT + def test_jira_issues_limited_to_visible_providers( + self, authenticated_client_rbac_limited, jira_issues_fixture + ): + linked, other_provider_issue, _ = jira_issues_fixture + response = authenticated_client_rbac_limited.get(reverse("jiraissue-list")) + assert response.status_code == status.HTTP_200_OK + assert [item["id"] for item in response.json()["data"]] == [str(linked.id)] + + response = authenticated_client_rbac_limited.get( + reverse("jiraissue-detail", kwargs={"pk": other_provider_issue.id}) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + def test_jira_issue_types_allowed_without_unlimited_visibility( self, authenticated_client_rbac_limited, jira_integration_fixture ): diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 7b153efb8f..0ff77d2820 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -34,6 +34,7 @@ from api.models import ( Integration, Invitation, InvitationRoleRelationship, + JiraIssue, LighthouseProviderConfiguration, LighthouseProviderModels, LighthouseTenantConfiguration, @@ -13557,6 +13558,144 @@ class TestScheduleViewSet: assert response.status_code == status.HTTP_409_CONFLICT +@pytest.mark.django_db +class TestJiraIssueViewSet: + def test_list_hides_reservations(self, authenticated_client, jira_issues_fixture): + linked, other_provider_issue, reservation = jira_issues_fixture + response = authenticated_client.get(reverse("jiraissue-list")) + assert response.status_code == status.HTTP_200_OK + ids = {item["id"] for item in response.json()["data"]} + assert ids == {str(linked.id), str(other_provider_issue.id)} + assert str(reservation.id) not in ids + + def test_retrieve(self, authenticated_client, jira_issues_fixture): + linked, *_ = jira_issues_fixture + response = authenticated_client.get( + reverse("jiraissue-detail", kwargs={"pk": linked.id}) + ) + assert response.status_code == status.HTTP_200_OK + data = response.json()["data"] + assert data["type"] == "jira-issues" + attributes = data["attributes"] + assert attributes["finding_uid"] == linked.finding_uid + assert attributes["finding_id"] == str(linked.finding_id) + assert attributes["issue_key"] == "TEST-1" + assert attributes["issue_url"] == "https://test.atlassian.net/browse/TEST-1" + assert attributes["project_key"] == "TEST" + assert attributes["issue_status"] == "To Do" + assert attributes["issue_status_category"] == "new" + assert attributes["status_synced_at"] is not None + relationships = data["relationships"] + assert relationships["provider"]["data"]["id"] == str(linked.provider_id) + assert relationships["integration"]["data"]["id"] == str(linked.integration_id) + + def test_retrieve_reservation_returns_404( + self, authenticated_client, jira_issues_fixture + ): + *_, reservation = jira_issues_fixture + response = authenticated_client.get( + reverse("jiraissue-detail", kwargs={"pk": reservation.id}) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_retrieve_other_tenant_returns_404( + self, authenticated_client, jira_issues_fixture, tenants_fixture + ): + linked, *_ = jira_issues_fixture + with rls_transaction(str(tenants_fixture[2].id)): + JiraIssue.objects.filter(id=linked.id).update( + tenant_id=tenants_fixture[2].id + ) + response = authenticated_client.get( + reverse("jiraissue-detail", kwargs={"pk": linked.id}) + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + @pytest.mark.parametrize( + "filter_name, filter_value, expected_keys", + [ + ("finding_uid", "test_finding_uid_1", {"TEST-1"}), + ( + "finding_uid__in", + "test_finding_uid_1,test_finding_uid_other_provider", + {"TEST-1", "TEST-2"}, + ), + ("finding_uid__in", "does-not-exist", set()), + ("issue_key", "TEST-2", {"TEST-2"}), + ("issue_status_category", "done", {"TEST-2"}), + ("issue_status_category__in", "new,indeterminate", {"TEST-1"}), + ("project_key", "TEST", {"TEST-1", "TEST-2"}), + ("search", "TEST-1", {"TEST-1"}), + ], + ) + def test_filters( + self, + authenticated_client, + jira_issues_fixture, + filter_name, + filter_value, + expected_keys, + ): + response = authenticated_client.get( + reverse("jiraissue-list"), {f"filter[{filter_name}]": filter_value} + ) + assert response.status_code == status.HTTP_200_OK + keys = {item["attributes"]["issue_key"] for item in response.json()["data"]} + assert keys == expected_keys + + def test_filter_by_provider(self, authenticated_client, jira_issues_fixture): + linked, other_provider_issue, _ = jira_issues_fixture + response = authenticated_client.get( + reverse("jiraissue-list"), + {"filter[provider_id]": str(other_provider_issue.provider_id)}, + ) + assert response.status_code == status.HTTP_200_OK + assert [item["id"] for item in response.json()["data"]] == [ + str(other_provider_issue.id) + ] + + def test_filter_by_integration_and_finding_id( + self, authenticated_client, jira_issues_fixture + ): + linked, *_ = jira_issues_fixture + response = authenticated_client.get( + reverse("jiraissue-list"), + { + "filter[integration]": str(linked.integration_id), + "filter[finding_id]": str(linked.finding_id), + }, + ) + assert response.status_code == status.HTTP_200_OK + assert [item["id"] for item in response.json()["data"]] == [str(linked.id)] + + def test_invalid_filter(self, authenticated_client, jira_issues_fixture): + response = authenticated_client.get( + reverse("jiraissue-list"), {"filter[invalid]": "x"} + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_include_provider(self, authenticated_client, jira_issues_fixture): + response = authenticated_client.get( + reverse("jiraissue-list"), {"include": "provider"} + ) + assert response.status_code == status.HTTP_200_OK + included_types = {item["type"] for item in response.json()["included"]} + assert included_types == {"providers"} + + def test_read_only(self, authenticated_client, jira_issues_fixture): + linked, *_ = jira_issues_fixture + response = authenticated_client.post( + reverse("jiraissue-list"), + data=json.dumps({"data": {"type": "jira-issues", "attributes": {}}}), + content_type="application/vnd.api+json", + ) + assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + response = authenticated_client.delete( + reverse("jiraissue-detail", kwargs={"pk": linked.id}) + ) + assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED + + @pytest.mark.django_db class TestIntegrationViewSet: def test_integrations_list(self, authenticated_client, integrations_fixture): diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 5e1f2fa6d8..ece4b37cae 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -14,6 +14,7 @@ from api.models import ( IntegrationProviderRelationship, Invitation, InvitationRoleRelationship, + JiraIssue, LighthouseConfiguration, LighthouseProviderConfiguration, LighthouseProviderModels, @@ -4149,6 +4150,41 @@ class LighthouseProviderModelsUpdateSerializer(BaseWriteSerializer): # Mute Rules +class JiraIssueSerializer(RLSSerializer): + """ + Read-only view of a Jira issue linked to a finding by a Jira integration. + + Rows are keyed on the finding ``uid`` so the same finding maps to the same + issue across scans. ``issue_status`` is the last status Prowler observed in + Jira (refreshed whenever a dispatch touches the finding), not a live value. + """ + + class Meta: + model = JiraIssue + fields = [ + "id", + "inserted_at", + "updated_at", + "finding_uid", + "finding_id", + "issue_key", + "issue_id", + "issue_url", + "project_key", + "issue_status", + "issue_status_category", + "status_synced_at", + "integration", + "provider", + "url", + ] + read_only_fields = fields + + included_serializers = { + "provider": "api.v1.serializers.ProviderIncludeSerializer", + } + + class MuteRuleSerializer(RLSSerializer): """ Serializer for reading MuteRule instances. diff --git a/api/src/backend/api/v1/urls.py b/api/src/backend/api/v1/urls.py index a558fa6387..077e1800b3 100644 --- a/api/src/backend/api/v1/urls.py +++ b/api/src/backend/api/v1/urls.py @@ -14,6 +14,7 @@ from api.v1.views import ( IntegrationViewSet, InvitationAcceptViewSet, InvitationViewSet, + JiraIssueViewSet, LighthouseConfigViewSet, LighthouseProviderConfigViewSet, LighthouseProviderModelsViewSet, @@ -107,6 +108,7 @@ router.register( basename="lighthouse-models", ) router.register(r"mute-rules", MuteRuleViewSet, basename="mute-rule") +router.register(r"jira-issues", JiraIssueViewSet, basename="jiraissue") tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant") tenants_router.register( diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 2717b523ac..e83aa6eb9a 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -54,6 +54,7 @@ from api.filters import ( IntegrationFilter, IntegrationJiraFindingsFilter, InvitationFilter, + JiraIssueFilter, LatestFindingFilter, LatestFindingGroupFilter, LatestFindingGroupSummaryFilter, @@ -89,6 +90,7 @@ from api.models import ( Integration, Invitation, InvitationRoleRelationship, + JiraIssue, LighthouseConfiguration, LighthouseProviderConfiguration, LighthouseProviderModels, @@ -179,6 +181,7 @@ from api.v1.serializers import ( InvitationCreateSerializer, InvitationSerializer, InvitationUpdateSerializer, + JiraIssueSerializer, LighthouseConfigCreateSerializer, LighthouseConfigSerializer, LighthouseConfigUpdateSerializer, @@ -7488,6 +7491,56 @@ class TenantApiKeyViewSet(BaseRLSViewSet): return Response(data=serializer.data, status=status.HTTP_200_OK) +# Jira issues +@extend_schema_view( + list=extend_schema( + tags=["Integration"], + summary="List Jira issues linked to findings", + description=( + "Retrieve the Jira issues created from findings through Jira integrations. " + "Each entry links a finding UID to the latest Jira issue created for it, " + "with the last status observed in Jira. Use `filter[finding_uid__in]` " + "and `filter[provider_id]` to check whether specific findings already " + "have a ticket." + ), + ), + retrieve=extend_schema( + tags=["Integration"], + summary="Retrieve a Jira issue link", + description="Fetch the Jira issue linked to a finding by the link ID.", + ), +) +class JiraIssueViewSet(BaseRLSViewSet): + queryset = JiraIssue.objects.all() + serializer_class = JiraIssueSerializer + filterset_class = JiraIssueFilter + http_method_names = ["get"] + search_fields = ["finding_uid", "issue_key"] + ordering = ["-inserted_at"] + ordering_fields = [ + "inserted_at", + "updated_at", + "issue_key", + "project_key", + "issue_status", + "status_synced_at", + ] + # RBAC required permissions (implicit -> MANAGE_PROVIDERS enables unlimited + # visibility or check visibility via provider group, like findings) + required_permissions = [] + + def get_queryset(self): + if getattr(self, "swagger_fake_view", False): + return JiraIssue.objects.none() + # Rows without an issue key are in-flight reservations, not links + queryset = JiraIssue.objects.filter(tenant_id=self.request.tenant_id).exclude( + issue_key="" + ) + if not self.user_role.unlimited_visibility: + queryset = queryset.filter(provider__in=get_providers(self.user_role)) + return queryset.select_related("provider", "integration") + + # MuteRules @extend_schema_view( list=extend_schema( diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index f2a3de5bd3..6e7c793dea 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -20,6 +20,7 @@ from api.models import ( Integration, IntegrationProviderRelationship, Invitation, + JiraIssue, LighthouseConfiguration, Membership, MuteRule, @@ -1470,6 +1471,52 @@ def jira_integration_fixture(tenants_fixture): ) +@pytest.fixture +def jira_issues_fixture(jira_integration_fixture, aws_provider_pair, findings_fixture): + """Two linked issues (one per provider) and one in-flight reservation.""" + provider, provider2 = aws_provider_pair + finding1, finding2 = findings_fixture + tenant_id = jira_integration_fixture.tenant_id + with rls_transaction(str(tenant_id)): + linked = JiraIssue.objects.create( + tenant_id=tenant_id, + integration=jira_integration_fixture, + provider=provider, + finding_uid=finding1.uid, + finding_id=finding1.id, + issue_key="TEST-1", + issue_id="10001", + issue_url="https://test.atlassian.net/browse/TEST-1", + project_key="TEST", + issue_status="To Do", + issue_status_category=JiraIssue.StatusCategoryChoices.NEW, + status_synced_at=datetime.now(UTC), + ) + hidden_provider_issue = JiraIssue.objects.create( + tenant_id=tenant_id, + integration=jira_integration_fixture, + provider=provider2, + finding_uid="test_finding_uid_other_provider", + finding_id=finding2.id, + issue_key="TEST-2", + issue_id="10002", + issue_url="https://test.atlassian.net/browse/TEST-2", + project_key="TEST", + issue_status="Done", + issue_status_category=JiraIssue.StatusCategoryChoices.DONE, + status_synced_at=datetime.now(UTC), + ) + reservation = JiraIssue.objects.create( + tenant_id=tenant_id, + integration=jira_integration_fixture, + provider=provider, + finding_uid=finding2.uid, + finding_id=finding2.id, + project_key="TEST", + ) + return linked, hidden_provider_issue, reservation + + @pytest.fixture def backfill_scan_metadata_fixture(scans_fixture, findings_fixture): for scan_instance in scans_fixture: diff --git a/api/src/backend/tasks/jobs/deletion.py b/api/src/backend/tasks/jobs/deletion.py index 91e64610f7..735d8bb48f 100644 --- a/api/src/backend/tasks/jobs/deletion.py +++ b/api/src/backend/tasks/jobs/deletion.py @@ -5,6 +5,7 @@ from api.db_utils import batch_delete, rls_transaction from api.models import ( AttackPathsScan, Finding, + JiraIssue, Provider, ProviderComplianceScore, Resource, @@ -86,6 +87,7 @@ def delete_provider(tenant_id: str, pk: str): deletion_steps = [ ("Scan Summaries", ScanSummary.all_objects.filter(scan__provider=instance)), + ("Jira Issues", JiraIssue.objects.filter(provider=instance)), ("Findings", Finding.all_objects.filter(scan__provider=instance)), ("Resources", Resource.all_objects.filter(provider=instance)), ("Scans", Scan.all_objects.filter(provider=instance)), diff --git a/api/src/backend/tasks/jobs/integrations.py b/api/src/backend/tasks/jobs/integrations.py index d8066d34be..f24d03db91 100644 --- a/api/src/backend/tasks/jobs/integrations.py +++ b/api/src/backend/tasks/jobs/integrations.py @@ -1,18 +1,19 @@ import os import time -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from glob import glob from urllib.parse import quote from api.db_router import READ_REPLICA_ALIAS, MainRouter from api.db_utils import REPLICA_MAX_ATTEMPTS, REPLICA_RETRY_BASE_DELAY, rls_transaction -from api.models import Finding, Integration, Provider +from api.models import Finding, Integration, JiraIssue, Provider from api.rls import Tenant from api.utils import initialize_prowler_integration, initialize_prowler_provider from celery.utils.log import get_task_logger from config.django.base import DJANGO_FINDINGS_BATCH_SIZE from django.conf import settings -from django.db import OperationalError +from django.db import IntegrityError, OperationalError +from django.utils import timezone from prowler.lib.outputs.asff.asff import ASFF from prowler.lib.outputs.compliance.generic.generic import GenericCompliance from prowler.lib.outputs.csv.csv import CSV @@ -555,6 +556,182 @@ def get_tenant_name(tenant_id: str) -> str: return "" +# Findings are pre-checked against existing Jira issues in chunks so the IN list +# stays bounded however many findings a dispatch carries +JIRA_DEDUP_CHUNK_SIZE = 500 +# A reservation (row without issue key) older than this belongs to a run that +# died mid-send and can be reclaimed +JIRA_RESERVATION_TTL = timedelta(minutes=15) +# Cap on the per-finding detail returned in the task result; counts are exact +JIRA_SKIPPED_REPORT_LIMIT = 100 + + +def _load_finding_refs(finding_ids: list[str]) -> dict[str, tuple[str, str]]: + """Map finding id -> (provider id, finding uid) for the batch, in one query.""" + refs = {} + for finding_id, provider_id, uid in Finding.all_objects.filter( + id__in=finding_ids + ).values_list("id", "scan__provider_id", "uid"): + refs[str(finding_id)] = (str(provider_id), uid) + return refs + + +def _load_existing_jira_issues( + tenant_id: str, integration_id: str, refs: dict[str, tuple[str, str]] +) -> dict[tuple[str, str], JiraIssue]: + """Load the Jira issue rows already linked to the batch's findings. + + Grouped by provider and chunked so each query is a bounded index lookup on + (tenant, integration, provider, finding_uid). + """ + uids_by_provider: dict[str, list[str]] = {} + for provider_id, uid in refs.values(): + uids_by_provider.setdefault(provider_id, []).append(uid) + + existing: dict[tuple[str, str], JiraIssue] = {} + for provider_id, uids in uids_by_provider.items(): + for start in range(0, len(uids), JIRA_DEDUP_CHUNK_SIZE): + chunk = uids[start : start + JIRA_DEDUP_CHUNK_SIZE] + for row in JiraIssue.objects.filter( + tenant_id=tenant_id, + integration_id=integration_id, + provider_id=provider_id, + finding_uid__in=chunk, + ): + existing[(str(row.provider_id), row.finding_uid)] = row + return existing + + +def _refresh_jira_issue_statuses( + tenant_id: str, jira_integration, rows: list[JiraIssue] +) -> dict[str, dict] | None: + """Fetch the current Jira status of linked rows and cache it on them. + + Returns the statuses keyed by issue key (keys missing from the result no + longer exist in Jira), or None when Jira could not be queried, in which case + the cached values are left untouched. + """ + keys = [row.issue_key for row in rows if row.issue_key] + if not keys: + return {} + try: + statuses = jira_integration.get_issues_status(keys) + except JiraBaseException as error: + logger.warning( + "Could not refresh Jira issue statuses, keeping cached values: %s", + error.message or error, + ) + return None + except Exception: + logger.exception("Could not refresh Jira issue statuses, keeping cached values") + return None + + now = timezone.now() + for row in rows: + status = statuses.get(row.issue_key) + if status is None: + # The issue is gone: keep the key for reference but mark it as done so + # the next send creates a fresh issue + row.issue_status = "" + row.issue_status_category = JiraIssue.StatusCategoryChoices.DONE + else: + row.issue_status = status.get("status", "")[:64] + row.issue_status_category = status.get("status_category", "")[:16] + row.status_synced_at = now + with rls_transaction(tenant_id): + JiraIssue.objects.bulk_update( + rows, ["issue_status", "issue_status_category", "status_synced_at"] + ) + return statuses + + +def _reserve_jira_issue( + tenant_id: str, + integration_id: str, + provider_id: str, + finding_uid: str, + finding_id: str, + project_key: str, +) -> JiraIssue | None: + """Claim the (integration, provider, finding uid) slot before calling Jira. + + The unique constraint makes this the arbiter between concurrent runs: only + the run that inserts the row (or reclaims an expired reservation) sends the + finding. Returns None when another run owns the slot. + """ + with rls_transaction(tenant_id): + try: + row, created = JiraIssue.objects.get_or_create( + tenant_id=tenant_id, + integration_id=integration_id, + provider_id=provider_id, + finding_uid=finding_uid, + defaults={"finding_id": finding_id, "project_key": project_key}, + ) + except IntegrityError: + return None + if created: + return row + if row.issue_key: + # Linked by a concurrent run between the pre-check and now + return None + if timezone.now() - row.updated_at < JIRA_RESERVATION_TTL: + # Another run is sending this finding right now + return None + # Expired reservation from a run that died mid-send: reclaim it + row.finding_id = finding_id + row.project_key = project_key + row.save(update_fields=["finding_id", "project_key", "updated_at"]) + return row + + +def _link_jira_issue( + tenant_id: str, row: JiraIssue, issue: dict, finding_id: str, project_key: str +) -> None: + """Point the row at the issue Jira just created.""" + with rls_transaction(tenant_id): + row.issue_key = (issue.get("key") or "")[:64] + row.issue_id = str(issue.get("id") or "")[:64] + row.issue_url = (issue.get("url") or "")[:2048] + row.project_key = project_key + row.finding_id = finding_id + row.issue_status = "" + row.issue_status_category = JiraIssue.StatusCategoryChoices.NEW + row.status_synced_at = None + row.save( + update_fields=[ + "issue_key", + "issue_id", + "issue_url", + "project_key", + "finding_id", + "issue_status", + "issue_status_category", + "status_synced_at", + "updated_at", + ] + ) + + +def _release_jira_issue(tenant_id: str, row: JiraIssue) -> None: + """Drop a reservation whose send failed so the finding can be retried.""" + if row.issue_key: + # A previously linked (now closed) issue stays linked; the send failed so + # there is nothing newer to point at + return + with rls_transaction(tenant_id): + JiraIssue.objects.filter(id=row.id, issue_key="").delete() + + +def _skipped_entry(finding_id: str, row: JiraIssue) -> dict: + return { + "finding_id": str(finding_id), + "issue_key": row.issue_key, + "issue_url": row.issue_url, + "issue_status": row.issue_status, + } + + def send_findings_to_jira( tenant_id: str, integration_id: str, @@ -562,14 +739,50 @@ def send_findings_to_jira( issue_type: str, finding_ids: list[str], ): + """Create one Jira issue per finding, skipping findings that already have one. + + Findings are matched to existing issues by (integration, provider, finding + uid), so a finding that was already sent in a previous scan is recognised. + Findings whose linked issue is still open are skipped and reported; findings + whose issue is closed or was deleted in Jira get a new issue that replaces + the link. + """ with rls_transaction(tenant_id): integration = Integration.objects.get(id=integration_id) jira_integration = initialize_prowler_integration(integration) tenant_info = get_tenant_name(tenant_id) + finding_refs = _load_finding_refs(finding_ids) + existing = _load_existing_jira_issues(tenant_id, integration_id, finding_refs) + + # Refresh the status of the linked issues in bulk so closed/deleted ones can be + # replaced. If Jira cannot be queried the linked findings are skipped as-is. + linked_rows = [row for row in existing.values() if row.issue_key] + statuses = ( + _refresh_jira_issue_statuses(tenant_id, jira_integration, linked_rows) + if linked_rows + else {} + ) num_tickets_created = 0 - error_messages = [] + skipped: list[dict] = [] + error_messages: list[str] = [] + created_rows: list[JiraIssue] = [] for finding_id in finding_ids: + finding_id = str(finding_id) + provider_id, finding_uid = finding_refs.get(finding_id, (None, None)) + row = existing.get((provider_id, finding_uid)) if provider_id else None + if row is not None: + if row.issue_key: + if statuses is None or not row.is_done: + # Still open (or status unknown): already ticketed + skipped.append(_skipped_entry(finding_id, row)) + continue + # Closed or deleted in Jira: create a replacement below + elif timezone.now() - row.updated_at < JIRA_RESERVATION_TTL: + # Another run is sending this finding right now + skipped.append(_skipped_entry(finding_id, row)) + continue + with rls_transaction(tenant_id): finding_instance = ( Finding.all_objects.select_related("scan__provider") @@ -599,69 +812,98 @@ def send_findings_to_jira( remediation_code = remediation.get("code", {}) provider_type = finding_instance.scan.provider.provider + if provider_id is None: + provider_id = str(finding_instance.scan.provider_id) + finding_uid = finding_instance.uid issue_labels = build_jira_issue_labels( - finding_uid=finding_instance.uid, + finding_uid=finding_uid, provider=provider_type, severity=finding_instance.severity, check_id=finding_instance.check_id, ) - finding_url = build_jira_finding_url(finding_instance.uid) + finding_url = build_jira_finding_url(finding_uid) - try: - # Send the individual finding to Jira - result = jira_integration.send_finding( - check_id=finding_instance.check_id, - check_title=check_metadata.get("checktitle", ""), - severity=finding_instance.severity, - status=finding_instance.status, - status_extended=finding_instance.status_extended or "", - provider=provider_type, - region=region, - resource_uid=resource_uid, - resource_name=resource_name, - risk=check_metadata.get("risk", ""), - recommendation_text=recommendation.get("text", ""), - recommendation_url=recommendation.get("url", ""), - remediation_code_native_iac=remediation_code.get("nativeiac", ""), - remediation_code_terraform=remediation_code.get("terraform", ""), - remediation_code_cli=remediation_code.get("cli", ""), - remediation_code_other=remediation_code.get("other", ""), - resource_tags=resource_tags, - compliance=finding_instance.compliance or {}, - project_key=project_key, - issue_type=issue_type, - issue_labels=issue_labels, - finding_url=finding_url, - tenant_info=tenant_info, - ) - except JiraBaseException as error: - error_message = error.message or JIRA_GENERIC_SEND_ERROR - logger.exception( - "Failed to send finding %s to Jira: %s", finding_id, error_message - ) - error_messages.append(error_message) - continue - except Exception: - logger.exception("Failed to send finding %s to Jira", finding_id) - error_messages.append(JIRA_GENERIC_SEND_ERROR) + if row is None: + row = _reserve_jira_issue( + tenant_id, + integration_id, + provider_id, + finding_uid, + finding_id, + project_key, + ) + if row is None: + skipped.append({"finding_id": finding_id}) continue - if result: - num_tickets_created += 1 - logger.info( - "Finding %s sent to Jira as %s", - finding_id, - result.get("key") if isinstance(result, dict) else result, - ) - else: - error_message = JIRA_GENERIC_SEND_ERROR - logger.error(error_message) - error_messages.append(error_message) + try: + # Send the individual finding to Jira + result = jira_integration.send_finding( + check_id=finding_instance.check_id, + check_title=check_metadata.get("checktitle", ""), + severity=finding_instance.severity, + status=finding_instance.status, + status_extended=finding_instance.status_extended or "", + provider=provider_type, + region=region, + resource_uid=resource_uid, + resource_name=resource_name, + risk=check_metadata.get("risk", ""), + recommendation_text=recommendation.get("text", ""), + recommendation_url=recommendation.get("url", ""), + remediation_code_native_iac=remediation_code.get("nativeiac", ""), + remediation_code_terraform=remediation_code.get("terraform", ""), + remediation_code_cli=remediation_code.get("cli", ""), + remediation_code_other=remediation_code.get("other", ""), + resource_tags=resource_tags, + compliance=finding_instance.compliance or {}, + project_key=project_key, + issue_type=issue_type, + issue_labels=issue_labels, + finding_url=finding_url, + tenant_info=tenant_info, + ) + except JiraBaseException as error: + error_message = error.message or JIRA_GENERIC_SEND_ERROR + logger.exception( + "Failed to send finding %s to Jira: %s", finding_id, error_message + ) + error_messages.append(error_message) + _release_jira_issue(tenant_id, row) + continue + except Exception: + logger.exception("Failed to send finding %s to Jira", finding_id) + error_messages.append(JIRA_GENERIC_SEND_ERROR) + _release_jira_issue(tenant_id, row) + continue + + if result: + num_tickets_created += 1 + issue = result if isinstance(result, dict) else {} + logger.info( + "Finding %s sent to Jira as %s", finding_id, issue.get("key") or result + ) + _link_jira_issue(tenant_id, row, issue, finding_id, project_key) + created_rows.append(row) + else: + error_message = JIRA_GENERIC_SEND_ERROR + logger.error(error_message) + error_messages.append(error_message) + _release_jira_issue(tenant_id, row) + + # Record the initial status of the issues just created, in bulk + if created_rows: + _refresh_jira_issue_statuses( + tenant_id, jira_integration, [row for row in created_rows if row.issue_key] + ) result = { "created_count": num_tickets_created, - "failed_count": len(finding_ids) - num_tickets_created, + "skipped_count": len(skipped), + "failed_count": len(finding_ids) - num_tickets_created - len(skipped), } + if skipped: + result["skipped"] = skipped[:JIRA_SKIPPED_REPORT_LIMIT] if error_messages: result["error"] = "; ".join(dict.fromkeys(error_messages)) diff --git a/api/src/backend/tasks/tests/test_deletion.py b/api/src/backend/tasks/tests/test_deletion.py index 9a6c4acc7c..3e785c65d1 100644 --- a/api/src/backend/tasks/tests/test_deletion.py +++ b/api/src/backend/tasks/tests/test_deletion.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, call, patch import pytest from api.attack_paths import database as graph_database -from api.models import Provider, Tenant, TenantComplianceSummary +from api.models import JiraIssue, Provider, Tenant, TenantComplianceSummary from django.core.exceptions import ObjectDoesNotExist from tasks.jobs.deletion import delete_provider, delete_tenant @@ -33,6 +33,22 @@ class TestDeleteProvider: str(instance.id), ) + def test_delete_provider_removes_jira_issues(self, jira_issues_fixture): + linked, other_provider_issue, reservation = jira_issues_fixture + provider = linked.provider + tenant_id = str(provider.tenant_id) + with ( + patch("tasks.jobs.deletion.graph_database.get_database_name"), + patch("tasks.jobs.deletion.graph_database.drop_subgraph"), + ): + delete_provider(tenant_id, provider.id) + + remaining = set(JiraIssue.objects.values_list("id", flat=True)) + assert linked.id not in remaining + assert reservation.id not in remaining + # Issues of other providers are untouched + assert other_provider_issue.id in remaining + def test_delete_provider_does_not_exist(self, tenants_fixture): with ( patch( diff --git a/api/src/backend/tasks/tests/test_integrations.py b/api/src/backend/tasks/tests/test_integrations.py index 2a66aa45a2..c1d60ffc35 100644 --- a/api/src/backend/tasks/tests/test_integrations.py +++ b/api/src/backend/tasks/tests/test_integrations.py @@ -1,12 +1,15 @@ -from datetime import UTC, datetime +import itertools +from datetime import UTC, datetime, timedelta from unittest.mock import MagicMock, patch import pytest from api.db_router import READ_REPLICA_ALIAS, MainRouter -from api.models import Integration +from api.db_utils import rls_transaction +from api.models import Integration, JiraIssue from api.utils import prowler_integration_connection_test from django.db import OperationalError from django.test import override_settings +from django.utils import timezone from prowler.lib.outputs.jira.exceptions.exceptions import ( JiraRefreshTokenError, JiraRequiredCustomFieldsError, @@ -14,6 +17,7 @@ from prowler.lib.outputs.jira.exceptions.exceptions import ( from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection from prowler.providers.common.models import Connection from tasks.jobs.integrations import ( + JIRA_RESERVATION_TTL, build_jira_finding_url, build_jira_issue_labels, get_s3_client_from_integration, @@ -1656,6 +1660,27 @@ class TestSecurityHubIntegrationUploads: @pytest.mark.django_db class TestJiraIntegration: + """Sending findings to Jira, with the dedup bookkeeping stubbed out. + + These tests use fake tenant ids and fully mocked findings; the dedup helpers + are exercised with real rows in TestJiraIssueDedup. + """ + + @pytest.fixture(autouse=True) + def no_dedup_bookkeeping(self): + reservation = MagicMock() + reservation.issue_key = "" + with patch.multiple( + "tasks.jobs.integrations", + _load_finding_refs=MagicMock(return_value={}), + _load_existing_jira_issues=MagicMock(return_value={}), + _refresh_jira_issue_statuses=MagicMock(return_value={}), + _reserve_jira_issue=MagicMock(return_value=reservation), + _link_jira_issue=MagicMock(), + _release_jira_issue=MagicMock(), + ): + yield + @patch("tasks.jobs.integrations.rls_transaction") @patch("tasks.jobs.integrations.Finding") @patch("tasks.jobs.integrations.Integration") @@ -1765,7 +1790,7 @@ class TestJiraIntegration: ) # Assertions - assert result == {"created_count": 2, "failed_count": 0} + assert result == {"created_count": 2, "skipped_count": 0, "failed_count": 0} # Verify Jira integration was initialized mock_initialize_integration.assert_called_once_with(integration) @@ -1885,6 +1910,7 @@ class TestJiraIntegration: # Assertions assert result == { "created_count": 2, + "skipped_count": 0, "failed_count": 1, "error": "Failed to create Jira issue.", } @@ -1951,6 +1977,7 @@ class TestJiraIntegration: assert result == { "created_count": 0, + "skipped_count": 0, "failed_count": 1, "error": error_message, } @@ -2019,6 +2046,7 @@ class TestJiraIntegration: assert result == { "created_count": 0, + "skipped_count": 0, "failed_count": 1, "error": error_message, } @@ -2083,6 +2111,7 @@ class TestJiraIntegration: assert result == { "created_count": 0, + "skipped_count": 0, "failed_count": 1, "error": "Failed to create Jira issue.", } @@ -2160,7 +2189,7 @@ class TestJiraIntegration: ) # Assertions - assert result == {"created_count": 1, "failed_count": 0} + assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0} # Verify send_finding was called with empty resource fields call_kwargs = mock_jira_integration.send_finding.call_args.kwargs @@ -2223,7 +2252,7 @@ class TestJiraIntegration: ) # Assertions - assert result == {"created_count": 1, "failed_count": 0} + assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0} # Verify send_finding was called with default/empty values call_kwargs = mock_jira_integration.send_finding.call_args.kwargs @@ -2314,3 +2343,282 @@ class TestJiraFindingReference: def test_get_tenant_name_unknown_or_invalid(self): assert get_tenant_name("00000000-0000-0000-0000-000000000000") == "" assert get_tenant_name("not-a-uuid") == "" + + +@pytest.mark.django_db +class TestJiraIssueDedup: + """send_findings_to_jira with real JiraIssue rows: skip, replace, reserve.""" + + @pytest.fixture + def jira_mock(self): + jira = MagicMock() + counter = itertools.count(1) + + def _create_issue(**kwargs): + number = next(counter) + return { + "key": f"TEST-{number}", + "id": str(10000 + number), + "url": f"https://test.atlassian.net/browse/TEST-{number}", + } + + jira.send_finding.side_effect = _create_issue + jira.get_issues_status.return_value = {} + return jira + + @pytest.fixture + def send(self, jira_mock, jira_integration_fixture): + def _send(finding_ids): + with patch( + "tasks.jobs.integrations.initialize_prowler_integration", + return_value=jira_mock, + ): + return send_findings_to_jira( + str(jira_integration_fixture.tenant_id), + str(jira_integration_fixture.id), + "TEST", + "Task", + [str(finding_id) for finding_id in finding_ids], + ) + + return _send + + @staticmethod + def _rows(integration): + with rls_transaction(str(integration.tenant_id)): + return { + row.finding_uid: row + for row in JiraIssue.objects.filter(integration=integration) + } + + def test_first_send_links_findings( + self, send, jira_mock, jira_integration_fixture, findings_fixture + ): + finding1, finding2 = findings_fixture + jira_mock.get_issues_status.return_value = { + "TEST-1": {"id": "10001", "status": "To Do", "status_category": "new"}, + "TEST-2": {"id": "10002", "status": "To Do", "status_category": "new"}, + } + + result = send([finding1.id, finding2.id]) + + assert result == {"created_count": 2, "skipped_count": 0, "failed_count": 0} + assert jira_mock.send_finding.call_count == 2 + rows = self._rows(jira_integration_fixture) + assert set(rows) == {finding1.uid, finding2.uid} + row = rows[finding1.uid] + assert row.issue_key == "TEST-1" + assert row.issue_id == "10001" + assert row.issue_url == "https://test.atlassian.net/browse/TEST-1" + assert row.project_key == "TEST" + assert row.finding_id == finding1.id + assert row.provider_id == finding1.scan.provider_id + # Status of the new issues is fetched once, in bulk, after creation + jira_mock.get_issues_status.assert_called_once_with(["TEST-1", "TEST-2"]) + assert row.issue_status == "To Do" + assert row.issue_status_category == "new" + assert row.status_synced_at is not None + + def test_second_send_skips_open_issue( + self, send, jira_mock, jira_integration_fixture, findings_fixture + ): + finding1, _ = findings_fixture + send([finding1.id]) + jira_mock.send_finding.reset_mock() + jira_mock.get_issues_status.reset_mock() + jira_mock.get_issues_status.return_value = { + "TEST-1": { + "id": "10001", + "status": "In Progress", + "status_category": "indeterminate", + } + } + + result = send([finding1.id]) + + assert result == { + "created_count": 0, + "skipped_count": 1, + "failed_count": 0, + "skipped": [ + { + "finding_id": str(finding1.id), + "issue_key": "TEST-1", + "issue_url": "https://test.atlassian.net/browse/TEST-1", + "issue_status": "In Progress", + } + ], + } + jira_mock.send_finding.assert_not_called() + jira_mock.get_issues_status.assert_called_once_with(["TEST-1"]) + row = self._rows(jira_integration_fixture)[finding1.uid] + assert row.issue_key == "TEST-1" + # The refreshed status is cached on the row + assert row.issue_status == "In Progress" + assert row.issue_status_category == "indeterminate" + + @pytest.mark.parametrize( + "status_response", + [ + {"TEST-1": {"id": "10001", "status": "Done", "status_category": "done"}}, + {}, # deleted in Jira + ], + ids=["closed", "deleted"], + ) + def test_closed_or_deleted_issue_is_replaced( + self, + send, + jira_mock, + jira_integration_fixture, + findings_fixture, + status_response, + ): + finding1, _ = findings_fixture + send([finding1.id]) + jira_mock.send_finding.reset_mock() + jira_mock.get_issues_status.return_value = status_response + + result = send([finding1.id]) + + assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0} + jira_mock.send_finding.assert_called_once() + rows = self._rows(jira_integration_fixture) + assert len(rows) == 1 + assert rows[finding1.uid].issue_key == "TEST-2" + assert ( + rows[finding1.uid].issue_url == "https://test.atlassian.net/browse/TEST-2" + ) + + def test_status_lookup_failure_skips_linked_findings( + self, send, jira_mock, jira_integration_fixture, findings_fixture + ): + finding1, _ = findings_fixture + send([finding1.id]) + jira_mock.send_finding.reset_mock() + jira_mock.get_issues_status.side_effect = JiraRefreshTokenError( + message="token expired" + ) + + result = send([finding1.id]) + + assert result["created_count"] == 0 + assert result["skipped_count"] == 1 + jira_mock.send_finding.assert_not_called() + # Cached status untouched + row = self._rows(jira_integration_fixture)[finding1.uid] + assert row.issue_key == "TEST-1" + + def test_failed_send_releases_reservation( + self, send, jira_mock, jira_integration_fixture, findings_fixture + ): + finding1, _ = findings_fixture + jira_mock.send_finding.side_effect = JiraRequiredCustomFieldsError( + message="custom fields" + ) + + result = send([finding1.id]) + + assert result["created_count"] == 0 + assert result["failed_count"] == 1 + assert result["error"] == "custom fields" + assert self._rows(jira_integration_fixture) == {} + + def test_failed_replacement_keeps_previous_link( + self, send, jira_mock, jira_integration_fixture, findings_fixture + ): + finding1, _ = findings_fixture + send([finding1.id]) + jira_mock.get_issues_status.return_value = { + "TEST-1": {"id": "10001", "status": "Done", "status_category": "done"} + } + jira_mock.send_finding.side_effect = Exception("boom") + + result = send([finding1.id]) + + assert result["failed_count"] == 1 + row = self._rows(jira_integration_fixture)[finding1.uid] + assert row.issue_key == "TEST-1" + assert row.issue_status_category == "done" + + def test_fresh_reservation_is_skipped_and_stale_one_reclaimed( + self, send, jira_mock, jira_integration_fixture, findings_fixture + ): + finding1, _ = findings_fixture + tenant_id = str(jira_integration_fixture.tenant_id) + with rls_transaction(tenant_id): + reservation = JiraIssue.objects.create( + tenant_id=tenant_id, + integration=jira_integration_fixture, + provider_id=finding1.scan.provider_id, + finding_uid=finding1.uid, + finding_id=finding1.id, + project_key="TEST", + ) + + # Another run is sending this finding right now + result = send([finding1.id]) + assert result == { + "created_count": 0, + "skipped_count": 1, + "failed_count": 0, + "skipped": [ + { + "finding_id": str(finding1.id), + "issue_key": "", + "issue_url": "", + "issue_status": "", + } + ], + } + jira_mock.send_finding.assert_not_called() + + # The run died: the reservation expired and is reclaimed + with rls_transaction(tenant_id): + JiraIssue.objects.filter(id=reservation.id).update( + updated_at=timezone.now() - JIRA_RESERVATION_TTL - timedelta(minutes=1) + ) + result = send([finding1.id]) + assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0} + rows = self._rows(jira_integration_fixture) + assert len(rows) == 1 + assert rows[finding1.uid].id == reservation.id + assert rows[finding1.uid].issue_key == "TEST-1" + + def test_other_integration_does_not_dedup( + self, + send, + jira_mock, + jira_integration_fixture, + findings_fixture, + tenants_fixture, + ): + finding1, _ = findings_fixture + tenant_id = str(jira_integration_fixture.tenant_id) + with rls_transaction(tenant_id): + other = Integration.objects.create( + tenant_id=tenant_id, + enabled=True, + connected=True, + integration_type=Integration.IntegrationChoices.JIRA, + configuration={"projects": {"OTHER": "Other"}}, + credentials={ + "domain": "other", + "user_mail": "a@b.com", + "api_token": "t", + }, + ) + send([finding1.id]) + jira_mock.send_finding.reset_mock() + + with patch( + "tasks.jobs.integrations.initialize_prowler_integration", + return_value=jira_mock, + ): + result = send_findings_to_jira( + tenant_id, str(other.id), "OTHER", "Task", [str(finding1.id)] + ) + + assert result["created_count"] == 1 + jira_mock.send_finding.assert_called_once() + with rls_transaction(tenant_id): + assert JiraIssue.objects.filter(finding_uid=finding1.uid).count() == 2 diff --git a/docs/user-guide/tutorials/prowler-app-jira-integration.mdx b/docs/user-guide/tutorials/prowler-app-jira-integration.mdx index 1df7f97bfc..7adec4e88d 100644 --- a/docs/user-guide/tutorials/prowler-app-jira-integration.mdx +++ b/docs/user-guide/tutorials/prowler-app-jira-integration.mdx @@ -134,6 +134,15 @@ Every Jira issue created from a single Finding carries a stable reference back t Prowler Cloud always includes the Finding URL. In a self-hosted Prowler App, set `DJANGO_UI_BASE_URL` in the API environment (for example, `https://prowler.example.com`) to enable it. When the variable is empty, the issue is created without the link. +### Sending a Finding That Already Has a Jira Issue + +Prowler remembers the Jira issue created for each Finding, keyed by the Finding UID, so sending the same Finding again does not create a duplicate issue: + +* If the linked issue is still open in Jira, the Finding is skipped and the existing issue key is reported in the task result (`skipped_count`, `skipped`). +* If the linked issue is closed (any Jira status in the **Done** category) or was deleted, a new issue is created and becomes the linked issue for that Finding. + +The link, and the last status observed in Jira, are available through the API at `GET /api/v1/jira-issues` (filter by `finding_uid`, `finding_uid__in`, `provider_id`, `integration` or `issue_key`). Each Jira integration keeps its own links, so the same Finding can have one issue per integration. + ## Integration Status Monitor and manage your Jira integrations through the management interface: