ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-10-05 03:12:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): track Jira issues per finding and skip already-ticketed findings

Browse Source 
New RLS table jira_issues keyed on (integration, provider, finding uid) that
stores the Jira issue key, URL and last observed status for every finding
sent to Jira. send_findings_to_jira() pre-checks the batch in bounded
chunks, refreshes the status of linked issues in bulk, skips findings whose
issue is still open (reported as skipped_count / skipped), creates a
replacement issue when the linked one is closed or deleted in Jira, and
reserves the slot before calling Jira so concurrent runs cannot duplicate.
Read-only GET /api/v1/jira-issues exposes the links with provider-scoped
visibility. Rows are removed with the provider.
This commit is contained in:
Daniel Barranquero
2026-08-25 16:58:40 +02:00
parent 53be73c2ac
commit 8097b8e012
18 changed files with 1741 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/changelog.d/jira-issue-dedup.added.md
+1
View File
@@ -0,0 +1 @@
Jira issues created from findings are now tracked per finding UID in the new `jira_issues` table and exposed through `GET /api/v1/jira-issues`; sending a finding that already has an open Jira issue skips it (reported as `skipped_count`), and findings whose issue was closed or deleted in Jira get a new issue that replaces the link
api/src/backend/api/filters.py
+28
View File
@@ -17,6 +17,7 @@ from api.models import (
FindingGroupDailySummary,
Integration,
Invitation,
JiraIssue,
LighthouseProviderConfiguration,
LighthouseProviderModels,
Membership,
@@ -1900,3 +1901,30 @@ class ComplianceWatchlistFilter(BaseProviderFilter):
class Meta(BaseProviderFilter.Meta):
model = ProviderComplianceScore
class JiraIssueFilter(BaseProviderFilter):
finding_uid = CharFilter(field_name="finding_uid", lookup_expr="exact")
finding_uid__in = CharInFilter(field_name="finding_uid", lookup_expr="in")
finding_id = UUIDFilter(field_name="finding_id", lookup_expr="exact")
finding_id__in = UUIDInFilter(field_name="finding_id", lookup_expr="in")
integration = UUIDFilter(field_name="integration__id", lookup_expr="exact")
integration__in = UUIDInFilter(field_name="integration__id", lookup_expr="in")
issue_key = CharFilter(field_name="issue_key", lookup_expr="exact")
issue_key__in = CharInFilter(field_name="issue_key", lookup_expr="in")
issue_status_category = ChoiceFilter(
choices=JiraIssue.StatusCategoryChoices.choices
)
issue_status_category__in = ChoiceInFilter(
choices=JiraIssue.StatusCategoryChoices.choices,
field_name="issue_status_category",
lookup_expr="in",
)
class Meta:
model = JiraIssue
fields = {
"inserted_at": ["date", "gte", "lte"],
"updated_at": ["date", "gte", "lte"],
"project_key": ["exact", "in"],
}
api/src/backend/api/migrations/0098_jira_issues.py
+104
View File
@@ -0,0 +1,104 @@
import uuid
import api.rls
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0097_attack_paths_scan_db_defaults"),
]
operations = [
migrations.CreateModel(
name="JiraIssue",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("inserted_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("finding_uid", models.CharField(max_length=300)),
("finding_id", models.UUIDField()),
(
"issue_key",
models.CharField(blank=True, default="", max_length=64),
),
(
"issue_id",
models.CharField(blank=True, default="", max_length=64),
),
(
"issue_url",
models.URLField(blank=True, default="", max_length=2048),
),
("project_key", models.CharField(max_length=64)),
(
"issue_status",
models.CharField(blank=True, default="", max_length=64),
),
(
"issue_status_category",
models.CharField(
blank=True,
choices=[
("new", "New"),
("indeterminate", "In progress"),
("done", "Done"),
],
default="",
max_length=16,
),
),
("status_synced_at", models.DateTimeField(blank=True, null=True)),
(
"integration",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="jira_issues",
to="api.integration",
),
),
(
"provider",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="jira_issues",
to="api.provider",
),
),
(
"tenant",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
),
),
],
options={
"db_table": "jira_issues",
"abstract": False,
},
),
migrations.AddConstraint(
model_name="jiraissue",
constraint=models.UniqueConstraint(
fields=("tenant_id", "integration_id", "provider_id", "finding_uid"),
name="unique_jira_issue_per_finding",
),
),
migrations.AddConstraint(
model_name="jiraissue",
constraint=api.rls.RowLevelSecurityConstraint(
"tenant_id",
name="rls_on_jiraissue",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
),
]
api/src/backend/api/migrations/0099_jira_issues_indexes.py
+17
View File
@@ -0,0 +1,17 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("api", "0098_jira_issues"),
]
operations = [
migrations.AddIndex(
model_name="jiraissue",
index=models.Index(
fields=["tenant_id", "provider_id", "finding_uid"],
name="ji_tenant_prov_uid_idx",
),
),
]
api/src/backend/api/models.py
+74
View File
@@ -3113,3 +3113,77 @@ class TenantComplianceSummary(RowLevelSecurityProtectedModel):
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
class JiraIssue(RowLevelSecurityProtectedModel):
"""Jira issue created from a finding through a Jira integration.
One row per (integration, provider, finding uid). Keyed on the finding ``uid``
rather than the per-scan finding id so the link survives rescans, which is
what lets a repeated send be recognised as already ticketed. Only the latest
ticket is kept: when a linked issue is closed or deleted in Jira and the
finding is sent again, the row is updated to point at the new issue.
"""
class StatusCategoryChoices(models.TextChoices):
NEW = "new", _("New")
INDETERMINATE = "indeterminate", _("In progress")
DONE = "done", _("Done")
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
updated_at = models.DateTimeField(auto_now=True, editable=False)
integration = models.ForeignKey(
Integration, on_delete=models.CASCADE, related_name="jira_issues"
)
provider = models.ForeignKey(
Provider, on_delete=models.CASCADE, related_name="jira_issues"
)
finding_uid = models.CharField(max_length=300)
# Last finding record that was sent; informational, findings are partitioned
# and rotate per scan so this is not a foreign key
finding_id = models.UUIDField()
# Empty while the issue is being created (reservation), filled after Jira
# confirms the creation
issue_key = models.CharField(max_length=64, blank=True, default="")
issue_id = models.CharField(max_length=64, blank=True, default="")
issue_url = models.URLField(max_length=2048, blank=True, default="")
project_key = models.CharField(max_length=64)
issue_status = models.CharField(max_length=64, blank=True, default="")
issue_status_category = models.CharField(
max_length=16, choices=StatusCategoryChoices.choices, blank=True, default=""
)
status_synced_at = models.DateTimeField(null=True, blank=True)
class Meta(RowLevelSecurityProtectedModel.Meta):
db_table = "jira_issues"
constraints = [
models.UniqueConstraint(
fields=("tenant_id", "integration_id", "provider_id", "finding_uid"),
name="unique_jira_issue_per_finding",
),
RowLevelSecurityConstraint(
field="tenant_id",
name="rls_on_%(class)s",
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
),
]
indexes = [
models.Index(
fields=["tenant_id", "provider_id", "finding_uid"],
name="ji_tenant_prov_uid_idx",
),
]
class JSONAPIMeta:
resource_name = "jira-issues"
@property
def is_linked(self) -> bool:
"""Whether the row points at a confirmed Jira issue (not a reservation)."""
return bool(self.issue_key)
@property
def is_done(self) -> bool:
return self.issue_status_category == self.StatusCategoryChoices.DONE
api/src/backend/api/specs/v1.yaml
+536
View File
@@ -7107,6 +7107,405 @@ paths:
schema:
$ref: '#/components/schemas/OpenApiResponseResponse'
description: ''
/api/v1/jira-issues:
get:
operationId: api_v1_jira_issues_list
description: Retrieve the Jira issues created from findings through Jira integrations.
Each entry links a finding UID to the latest Jira issue created for it, with
the last status observed in Jira. Use `filter[finding_uid__in]` and `filter[provider_id]`
to check whether specific findings already have a ticket.
summary: List Jira issues linked to findings
parameters:
- in: query
name: fields[jira-issues]
schema:
type: array
items:
type: string
enum:
- inserted_at
- updated_at
- finding_uid
- finding_id
- issue_key
- issue_id
- issue_url
- project_key
- issue_status
- issue_status_category
- status_synced_at
- integration
- provider
- url
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: query
name: filter[finding_id]
schema:
type: string
format: uuid
- in: query
name: filter[finding_id__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[finding_uid]
schema:
type: string
- in: query
name: filter[finding_uid__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[inserted_at__date]
schema:
type: string
format: date
- in: query
name: filter[inserted_at__gte]
schema:
type: string
format: date-time
- in: query
name: filter[inserted_at__lte]
schema:
type: string
format: date-time
- in: query
name: filter[integration]
schema:
type: string
format: uuid
- in: query
name: filter[integration__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[issue_key]
schema:
type: string
- in: query
name: filter[issue_key__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[issue_status_category]
schema:
type: string
x-spec-enum-id: 6e5c623f6bbdd92d
enum:
- done
- indeterminate
- new
description: |-
* `new` - New
* `indeterminate` - In progress
* `done` - Done
- in: query
name: filter[issue_status_category__in]
schema:
type: array
items:
type: string
x-spec-enum-id: 6e5c623f6bbdd92d
enum:
- done
- indeterminate
- new
description: |-
Multiple values may be separated by commas.
* `new` - New
* `indeterminate` - In progress
* `done` - Done
explode: false
style: form
- in: query
name: filter[project_key]
schema:
type: string
- in: query
name: filter[project_key__in]
schema:
type: array
items:
type: string
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_groups]
schema:
type: string
format: uuid
- in: query
name: filter[provider_groups__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_id]
schema:
type: string
format: uuid
- in: query
name: filter[provider_id__in]
schema:
type: array
items:
type: string
format: uuid
description: Multiple values may be separated by commas.
explode: false
style: form
- in: query
name: filter[provider_type]
schema:
type: string
x-spec-enum-id: 203afc16daac9b64
enum:
- alibabacloud
- aws
- azure
- cloudflare
- gcp
- github
- googleworkspace
- iac
- image
- kubernetes
- m365
- mongodbatlas
- okta
- openstack
- oraclecloud
- vercel
description: |-
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
* `m365` - M365
* `github` - GitHub
* `mongodbatlas` - MongoDB Atlas
* `iac` - IaC
* `oraclecloud` - Oracle Cloud Infrastructure
* `alibabacloud` - Alibaba Cloud
* `cloudflare` - Cloudflare
* `openstack` - OpenStack
* `image` - Image
* `googleworkspace` - Google Workspace
* `vercel` - Vercel
* `okta` - Okta
- in: query
name: filter[provider_type__in]
schema:
type: array
items:
type: string
x-spec-enum-id: 203afc16daac9b64
enum:
- alibabacloud
- aws
- azure
- cloudflare
- gcp
- github
- googleworkspace
- iac
- image
- kubernetes
- m365
- mongodbatlas
- okta
- openstack
- oraclecloud
- vercel
description: |-
Multiple values may be separated by commas.
* `aws` - AWS
* `azure` - Azure
* `gcp` - GCP
* `kubernetes` - Kubernetes
* `m365` - M365
* `github` - GitHub
* `mongodbatlas` - MongoDB Atlas
* `iac` - IaC
* `oraclecloud` - Oracle Cloud Infrastructure
* `alibabacloud` - Alibaba Cloud
* `cloudflare` - Cloudflare
* `openstack` - OpenStack
* `image` - Image
* `googleworkspace` - Google Workspace
* `vercel` - Vercel
* `okta` - Okta
explode: false
style: form
- name: filter[search]
required: false
in: query
description: A search term.
schema:
type: string
- in: query
name: filter[updated_at__date]
schema:
type: string
format: date
- in: query
name: filter[updated_at__gte]
schema:
type: string
format: date-time
- in: query
name: filter[updated_at__lte]
schema:
type: string
format: date-time
- in: query
name: include
schema:
type: array
items:
type: string
enum:
- provider
description: include query parameter to allow the client to customize which
related resources should be returned.
explode: false
- name: page[number]
required: false
in: query
description: A page number within the paginated result set.
schema:
type: integer
- name: page[size]
required: false
in: query
description: Number of results to return per page.
schema:
type: integer
- name: sort
required: false
in: query
description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)'
schema:
type: array
items:
type: string
enum:
- inserted_at
- -inserted_at
- updated_at
- -updated_at
- issue_key
- -issue_key
- project_key
- -project_key
- issue_status
- -issue_status
- status_synced_at
- -status_synced_at
explode: false
tags:
- Integration
security:
- JWT or API Key: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/PaginatedJiraIssueList'
description: ''
/api/v1/jira-issues/{id}:
get:
operationId: api_v1_jira_issues_retrieve
description: Fetch the Jira issue linked to a finding by the link ID.
summary: Retrieve a Jira issue link
parameters:
- in: query
name: fields[jira-issues]
schema:
type: array
items:
type: string
enum:
- inserted_at
- updated_at
- finding_uid
- finding_id
- issue_key
- issue_id
- issue_url
- project_key
- issue_status
- issue_status_category
- status_synced_at
- integration
- provider
- url
description: endpoint return only specific fields in the response on a per-type
basis by including a fields[TYPE] query parameter.
explode: false
- in: path
name: id
schema:
type: string
format: uuid
description: A UUID string identifying this jira issue.
required: true
- in: query
name: include
schema:
type: array
items:
type: string
enum:
- provider
description: include query parameter to allow the client to customize which
related resources should be returned.
explode: false
tags:
- Integration
security:
- JWT or API Key: []
responses:
'200':
content:
application/vnd.api+json:
schema:
$ref: '#/components/schemas/JiraIssueResponse'
description: ''
/api/v1/lighthouse-configurations:
get:
operationId: api_v1_lighthouse_configurations_list
@@ -18618,6 +19017,134 @@ components:
$ref: '#/components/schemas/InvitationUpdate'
required:
- data
JiraIssue:
type: object
required:
- type
- id
additionalProperties: false
properties:
type:
type: string
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common attributes
and relationships.
enum:
- jira-issues
id:
type: string
format: uuid
attributes:
type: object
properties:
inserted_at:
type: string
format: date-time
readOnly: true
updated_at:
type: string
format: date-time
readOnly: true
finding_uid:
type: string
readOnly: true
finding_id:
type: string
format: uuid
readOnly: true
issue_key:
type: string
readOnly: true
issue_id:
type: string
readOnly: true
issue_url:
type: string
format: uri
readOnly: true
project_key:
type: string
readOnly: true
issue_status:
type: string
readOnly: true
issue_status_category:
enum:
- new
- indeterminate
- done
type: string
description: |-
* `new` - New
* `indeterminate` - In progress
* `done` - Done
x-spec-enum-id: 6e5c623f6bbdd92d
readOnly: true
status_synced_at:
type: string
format: date-time
readOnly: true
nullable: true
relationships:
type: object
properties:
integration:
type: object
properties:
data:
type: object
properties:
id:
type: string
format: uuid
type:
type: string
enum:
- integrations
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
attributes and relationships.
required:
- id
- type
required:
- data
description: The identifier of the related object.
title: Resource Identifier
readOnly: true
provider:
type: object
properties:
data:
type: object
properties:
id:
type: string
format: uuid
type:
type: string
enum:
- providers
title: Resource Type Name
description: The [type](https://jsonapi.org/format/#document-resource-object-identification)
member is used to describe resource objects that share common
attributes and relationships.
required:
- id
- type
required:
- data
description: The identifier of the related object.
title: Resource Identifier
readOnly: true
JiraIssueResponse:
type: object
properties:
data:
$ref: '#/components/schemas/JiraIssue'
required:
- data
LighthouseConfig:
type: object
required:
@@ -20227,6 +20754,15 @@ components:
$ref: '#/components/schemas/Invitation'
required:
- data
PaginatedJiraIssueList:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/JiraIssue'
required:
- data
PaginatedLighthouseConfigList:
type: object
properties:
api/src/backend/api/tests/test_models.py
+53
View File
@@ -4,6 +4,7 @@ import pytest
from allauth.socialaccount.models import SocialApp
from api.db_router import MainRouter
from api.models import (
JiraIssue,
ProviderComplianceScore,
Resource,
ResourceTag,
@@ -524,3 +525,55 @@ class TestTenantComplianceSummaryModel:
assert summary1.id != summary2.id
assert summary1.requirements_passed != summary2.requirements_passed
@pytest.mark.django_db
class TestJiraIssueModel:
def test_create_jira_issue(
self, jira_integration_fixture, aws_provider, findings_fixture
):
finding = findings_fixture[0]
issue = JiraIssue.objects.create(
tenant_id=jira_integration_fixture.tenant_id,
integration=jira_integration_fixture,
provider=aws_provider,
finding_uid=finding.uid,
finding_id=finding.id,
issue_key="TEST-1",
project_key="TEST",
)
assert issue.is_linked
assert not issue.is_done
assert issue.issue_status_category == ""
def test_reservation_is_not_linked(
self, jira_integration_fixture, aws_provider, findings_fixture
):
finding = findings_fixture[0]
issue = JiraIssue.objects.create(
tenant_id=jira_integration_fixture.tenant_id,
integration=jira_integration_fixture,
provider=aws_provider,
finding_uid=finding.uid,
finding_id=finding.id,
project_key="TEST",
)
assert not issue.is_linked
def test_unique_per_integration_provider_and_finding_uid(
self, jira_integration_fixture, aws_provider_pair, findings_fixture
):
provider, provider2 = aws_provider_pair
finding = findings_fixture[0]
common = {
"tenant_id": jira_integration_fixture.tenant_id,
"integration": jira_integration_fixture,
"finding_uid": finding.uid,
"finding_id": finding.id,
"project_key": "TEST",
}
JiraIssue.objects.create(provider=provider, issue_key="TEST-1", **common)
# Same finding uid on another provider is a different finding
JiraIssue.objects.create(provider=provider2, issue_key="TEST-2", **common)
with pytest.raises(IntegrityError):
JiraIssue.objects.create(provider=provider, issue_key="TEST-3", **common)
api/src/backend/api/tests/test_rbac.py
+13
View File
@@ -1591,6 +1591,19 @@ class TestLimitedVisibility:
assert response.status_code == status.HTTP_204_NO_CONTENT
def test_jira_issues_limited_to_visible_providers(
self, authenticated_client_rbac_limited, jira_issues_fixture
):
linked, other_provider_issue, _ = jira_issues_fixture
response = authenticated_client_rbac_limited.get(reverse("jiraissue-list"))
assert response.status_code == status.HTTP_200_OK
assert [item["id"] for item in response.json()["data"]] == [str(linked.id)]
response = authenticated_client_rbac_limited.get(
reverse("jiraissue-detail", kwargs={"pk": other_provider_issue.id})
)
assert response.status_code == status.HTTP_404_NOT_FOUND
def test_jira_issue_types_allowed_without_unlimited_visibility(
self, authenticated_client_rbac_limited, jira_integration_fixture
):
api/src/backend/api/tests/test_views.py
+139
View File
@@ -34,6 +34,7 @@ from api.models import (
Integration,
Invitation,
InvitationRoleRelationship,
JiraIssue,
LighthouseProviderConfiguration,
LighthouseProviderModels,
LighthouseTenantConfiguration,
@@ -13557,6 +13558,144 @@ class TestScheduleViewSet:
assert response.status_code == status.HTTP_409_CONFLICT
@pytest.mark.django_db
class TestJiraIssueViewSet:
def test_list_hides_reservations(self, authenticated_client, jira_issues_fixture):
linked, other_provider_issue, reservation = jira_issues_fixture
response = authenticated_client.get(reverse("jiraissue-list"))
assert response.status_code == status.HTTP_200_OK
ids = {item["id"] for item in response.json()["data"]}
assert ids == {str(linked.id), str(other_provider_issue.id)}
assert str(reservation.id) not in ids
def test_retrieve(self, authenticated_client, jira_issues_fixture):
linked, *_ = jira_issues_fixture
response = authenticated_client.get(
reverse("jiraissue-detail", kwargs={"pk": linked.id})
)
assert response.status_code == status.HTTP_200_OK
data = response.json()["data"]
assert data["type"] == "jira-issues"
attributes = data["attributes"]
assert attributes["finding_uid"] == linked.finding_uid
assert attributes["finding_id"] == str(linked.finding_id)
assert attributes["issue_key"] == "TEST-1"
assert attributes["issue_url"] == "https://test.atlassian.net/browse/TEST-1"
assert attributes["project_key"] == "TEST"
assert attributes["issue_status"] == "To Do"
assert attributes["issue_status_category"] == "new"
assert attributes["status_synced_at"] is not None
relationships = data["relationships"]
assert relationships["provider"]["data"]["id"] == str(linked.provider_id)
assert relationships["integration"]["data"]["id"] == str(linked.integration_id)
def test_retrieve_reservation_returns_404(
self, authenticated_client, jira_issues_fixture
):
*_, reservation = jira_issues_fixture
response = authenticated_client.get(
reverse("jiraissue-detail", kwargs={"pk": reservation.id})
)
assert response.status_code == status.HTTP_404_NOT_FOUND
def test_retrieve_other_tenant_returns_404(
self, authenticated_client, jira_issues_fixture, tenants_fixture
):
linked, *_ = jira_issues_fixture
with rls_transaction(str(tenants_fixture[2].id)):
JiraIssue.objects.filter(id=linked.id).update(
tenant_id=tenants_fixture[2].id
)
response = authenticated_client.get(
reverse("jiraissue-detail", kwargs={"pk": linked.id})
)
assert response.status_code == status.HTTP_404_NOT_FOUND
@pytest.mark.parametrize(
"filter_name, filter_value, expected_keys",
[
("finding_uid", "test_finding_uid_1", {"TEST-1"}),
(
"finding_uid__in",
"test_finding_uid_1,test_finding_uid_other_provider",
{"TEST-1", "TEST-2"},
),
("finding_uid__in", "does-not-exist", set()),
("issue_key", "TEST-2", {"TEST-2"}),
("issue_status_category", "done", {"TEST-2"}),
("issue_status_category__in", "new,indeterminate", {"TEST-1"}),
("project_key", "TEST", {"TEST-1", "TEST-2"}),
("search", "TEST-1", {"TEST-1"}),
],
)
def test_filters(
self,
authenticated_client,
jira_issues_fixture,
filter_name,
filter_value,
expected_keys,
):
response = authenticated_client.get(
reverse("jiraissue-list"), {f"filter[{filter_name}]": filter_value}
)
assert response.status_code == status.HTTP_200_OK
keys = {item["attributes"]["issue_key"] for item in response.json()["data"]}
assert keys == expected_keys
def test_filter_by_provider(self, authenticated_client, jira_issues_fixture):
linked, other_provider_issue, _ = jira_issues_fixture
response = authenticated_client.get(
reverse("jiraissue-list"),
{"filter[provider_id]": str(other_provider_issue.provider_id)},
)
assert response.status_code == status.HTTP_200_OK
assert [item["id"] for item in response.json()["data"]] == [
str(other_provider_issue.id)
]
def test_filter_by_integration_and_finding_id(
self, authenticated_client, jira_issues_fixture
):
linked, *_ = jira_issues_fixture
response = authenticated_client.get(
reverse("jiraissue-list"),
{
"filter[integration]": str(linked.integration_id),
"filter[finding_id]": str(linked.finding_id),
},
)
assert response.status_code == status.HTTP_200_OK
assert [item["id"] for item in response.json()["data"]] == [str(linked.id)]
def test_invalid_filter(self, authenticated_client, jira_issues_fixture):
response = authenticated_client.get(
reverse("jiraissue-list"), {"filter[invalid]": "x"}
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
def test_include_provider(self, authenticated_client, jira_issues_fixture):
response = authenticated_client.get(
reverse("jiraissue-list"), {"include": "provider"}
)
assert response.status_code == status.HTTP_200_OK
included_types = {item["type"] for item in response.json()["included"]}
assert included_types == {"providers"}
def test_read_only(self, authenticated_client, jira_issues_fixture):
linked, *_ = jira_issues_fixture
response = authenticated_client.post(
reverse("jiraissue-list"),
data=json.dumps({"data": {"type": "jira-issues", "attributes": {}}}),
content_type="application/vnd.api+json",
)
assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED
response = authenticated_client.delete(
reverse("jiraissue-detail", kwargs={"pk": linked.id})
)
assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED
@pytest.mark.django_db
class TestIntegrationViewSet:
def test_integrations_list(self, authenticated_client, integrations_fixture):
api/src/backend/api/v1/serializers.py
+36
View File
@@ -14,6 +14,7 @@ from api.models import (
IntegrationProviderRelationship,
Invitation,
InvitationRoleRelationship,
JiraIssue,
LighthouseConfiguration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
@@ -4149,6 +4150,41 @@ class LighthouseProviderModelsUpdateSerializer(BaseWriteSerializer):
# Mute Rules
class JiraIssueSerializer(RLSSerializer):
"""
Read-only view of a Jira issue linked to a finding by a Jira integration.
Rows are keyed on the finding ``uid`` so the same finding maps to the same
issue across scans. ``issue_status`` is the last status Prowler observed in
Jira (refreshed whenever a dispatch touches the finding), not a live value.
"""
class Meta:
model = JiraIssue
fields = [
"id",
"inserted_at",
"updated_at",
"finding_uid",
"finding_id",
"issue_key",
"issue_id",
"issue_url",
"project_key",
"issue_status",
"issue_status_category",
"status_synced_at",
"integration",
"provider",
"url",
]
read_only_fields = fields
included_serializers = {
"provider": "api.v1.serializers.ProviderIncludeSerializer",
}
class MuteRuleSerializer(RLSSerializer):
"""
Serializer for reading MuteRule instances.
api/src/backend/api/v1/urls.py
+2
View File
@@ -14,6 +14,7 @@ from api.v1.views import (
IntegrationViewSet,
InvitationAcceptViewSet,
InvitationViewSet,
JiraIssueViewSet,
LighthouseConfigViewSet,
LighthouseProviderConfigViewSet,
LighthouseProviderModelsViewSet,
@@ -107,6 +108,7 @@ router.register(
basename="lighthouse-models",
)
router.register(r"mute-rules", MuteRuleViewSet, basename="mute-rule")
router.register(r"jira-issues", JiraIssueViewSet, basename="jiraissue")
tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant")
tenants_router.register(
api/src/backend/api/v1/views.py
+53
View File
@@ -54,6 +54,7 @@ from api.filters import (
IntegrationFilter,
IntegrationJiraFindingsFilter,
InvitationFilter,
JiraIssueFilter,
LatestFindingFilter,
LatestFindingGroupFilter,
LatestFindingGroupSummaryFilter,
@@ -89,6 +90,7 @@ from api.models import (
Integration,
Invitation,
InvitationRoleRelationship,
JiraIssue,
LighthouseConfiguration,
LighthouseProviderConfiguration,
LighthouseProviderModels,
@@ -179,6 +181,7 @@ from api.v1.serializers import (
InvitationCreateSerializer,
InvitationSerializer,
InvitationUpdateSerializer,
JiraIssueSerializer,
LighthouseConfigCreateSerializer,
LighthouseConfigSerializer,
LighthouseConfigUpdateSerializer,
@@ -7488,6 +7491,56 @@ class TenantApiKeyViewSet(BaseRLSViewSet):
return Response(data=serializer.data, status=status.HTTP_200_OK)
# Jira issues
@extend_schema_view(
list=extend_schema(
tags=["Integration"],
summary="List Jira issues linked to findings",
description=(
"Retrieve the Jira issues created from findings through Jira integrations. "
"Each entry links a finding UID to the latest Jira issue created for it, "
"with the last status observed in Jira. Use `filter[finding_uid__in]` "
"and `filter[provider_id]` to check whether specific findings already "
"have a ticket."
),
),
retrieve=extend_schema(
tags=["Integration"],
summary="Retrieve a Jira issue link",
description="Fetch the Jira issue linked to a finding by the link ID.",
),
)
class JiraIssueViewSet(BaseRLSViewSet):
queryset = JiraIssue.objects.all()
serializer_class = JiraIssueSerializer
filterset_class = JiraIssueFilter
http_method_names = ["get"]
search_fields = ["finding_uid", "issue_key"]
ordering = ["-inserted_at"]
ordering_fields = [
"inserted_at",
"updated_at",
"issue_key",
"project_key",
"issue_status",
"status_synced_at",
]
# RBAC required permissions (implicit -> MANAGE_PROVIDERS enables unlimited
# visibility or check visibility via provider group, like findings)
required_permissions = []
def get_queryset(self):
if getattr(self, "swagger_fake_view", False):
return JiraIssue.objects.none()
# Rows without an issue key are in-flight reservations, not links
queryset = JiraIssue.objects.filter(tenant_id=self.request.tenant_id).exclude(
issue_key=""
)
if not self.user_role.unlimited_visibility:
queryset = queryset.filter(provider__in=get_providers(self.user_role))
return queryset.select_related("provider", "integration")
# MuteRules
@extend_schema_view(
list=extend_schema(
api/src/backend/conftest.py
+47
View File
@@ -20,6 +20,7 @@ from api.models import (
Integration,
IntegrationProviderRelationship,
Invitation,
JiraIssue,
LighthouseConfiguration,
Membership,
MuteRule,
@@ -1470,6 +1471,52 @@ def jira_integration_fixture(tenants_fixture):
)
@pytest.fixture
def jira_issues_fixture(jira_integration_fixture, aws_provider_pair, findings_fixture):
"""Two linked issues (one per provider) and one in-flight reservation."""
provider, provider2 = aws_provider_pair
finding1, finding2 = findings_fixture
tenant_id = jira_integration_fixture.tenant_id
with rls_transaction(str(tenant_id)):
linked = JiraIssue.objects.create(
tenant_id=tenant_id,
integration=jira_integration_fixture,
provider=provider,
finding_uid=finding1.uid,
finding_id=finding1.id,
issue_key="TEST-1",
issue_id="10001",
issue_url="https://test.atlassian.net/browse/TEST-1",
project_key="TEST",
issue_status="To Do",
issue_status_category=JiraIssue.StatusCategoryChoices.NEW,
status_synced_at=datetime.now(UTC),
)
hidden_provider_issue = JiraIssue.objects.create(
tenant_id=tenant_id,
integration=jira_integration_fixture,
provider=provider2,
finding_uid="test_finding_uid_other_provider",
finding_id=finding2.id,
issue_key="TEST-2",
issue_id="10002",
issue_url="https://test.atlassian.net/browse/TEST-2",
project_key="TEST",
issue_status="Done",
issue_status_category=JiraIssue.StatusCategoryChoices.DONE,
status_synced_at=datetime.now(UTC),
)
reservation = JiraIssue.objects.create(
tenant_id=tenant_id,
integration=jira_integration_fixture,
provider=provider,
finding_uid=finding2.uid,
finding_id=finding2.id,
project_key="TEST",
)
return linked, hidden_provider_issue, reservation
@pytest.fixture
def backfill_scan_metadata_fixture(scans_fixture, findings_fixture):
for scan_instance in scans_fixture:
api/src/backend/tasks/jobs/deletion.py
+2
View File
@@ -5,6 +5,7 @@ from api.db_utils import batch_delete, rls_transaction
from api.models import (
AttackPathsScan,
Finding,
JiraIssue,
Provider,
ProviderComplianceScore,
Resource,
@@ -86,6 +87,7 @@ def delete_provider(tenant_id: str, pk: str):
deletion_steps = [
("Scan Summaries", ScanSummary.all_objects.filter(scan__provider=instance)),
("Jira Issues", JiraIssue.objects.filter(provider=instance)),
("Findings", Finding.all_objects.filter(scan__provider=instance)),
("Resources", Resource.all_objects.filter(provider=instance)),
("Scans", Scan.all_objects.filter(provider=instance)),
api/src/backend/tasks/jobs/integrations.py
+297 -55
View File
@@ -1,18 +1,19 @@
import os
import time
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from glob import glob
from urllib.parse import quote
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.db_utils import REPLICA_MAX_ATTEMPTS, REPLICA_RETRY_BASE_DELAY, rls_transaction
from api.models import Finding, Integration, Provider
from api.models import Finding, Integration, JiraIssue, Provider
from api.rls import Tenant
from api.utils import initialize_prowler_integration, initialize_prowler_provider
from celery.utils.log import get_task_logger
from config.django.base import DJANGO_FINDINGS_BATCH_SIZE
from django.conf import settings
from django.db import OperationalError
from django.db import IntegrityError, OperationalError
from django.utils import timezone
from prowler.lib.outputs.asff.asff import ASFF
from prowler.lib.outputs.compliance.generic.generic import GenericCompliance
from prowler.lib.outputs.csv.csv import CSV
@@ -555,6 +556,182 @@ def get_tenant_name(tenant_id: str) -> str:
return ""
# Findings are pre-checked against existing Jira issues in chunks so the IN list
# stays bounded however many findings a dispatch carries
JIRA_DEDUP_CHUNK_SIZE = 500
# A reservation (row without issue key) older than this belongs to a run that
# died mid-send and can be reclaimed
JIRA_RESERVATION_TTL = timedelta(minutes=15)
# Cap on the per-finding detail returned in the task result; counts are exact
JIRA_SKIPPED_REPORT_LIMIT = 100
def _load_finding_refs(finding_ids: list[str]) -> dict[str, tuple[str, str]]:
"""Map finding id -> (provider id, finding uid) for the batch, in one query."""
refs = {}
for finding_id, provider_id, uid in Finding.all_objects.filter(
id__in=finding_ids
).values_list("id", "scan__provider_id", "uid"):
refs[str(finding_id)] = (str(provider_id), uid)
return refs
def _load_existing_jira_issues(
tenant_id: str, integration_id: str, refs: dict[str, tuple[str, str]]
) -> dict[tuple[str, str], JiraIssue]:
"""Load the Jira issue rows already linked to the batch's findings.
Grouped by provider and chunked so each query is a bounded index lookup on
(tenant, integration, provider, finding_uid).
"""
uids_by_provider: dict[str, list[str]] = {}
for provider_id, uid in refs.values():
uids_by_provider.setdefault(provider_id, []).append(uid)
existing: dict[tuple[str, str], JiraIssue] = {}
for provider_id, uids in uids_by_provider.items():
for start in range(0, len(uids), JIRA_DEDUP_CHUNK_SIZE):
chunk = uids[start : start + JIRA_DEDUP_CHUNK_SIZE]
for row in JiraIssue.objects.filter(
tenant_id=tenant_id,
integration_id=integration_id,
provider_id=provider_id,
finding_uid__in=chunk,
):
existing[(str(row.provider_id), row.finding_uid)] = row
return existing
def _refresh_jira_issue_statuses(
tenant_id: str, jira_integration, rows: list[JiraIssue]
) -> dict[str, dict] | None:
"""Fetch the current Jira status of linked rows and cache it on them.
Returns the statuses keyed by issue key (keys missing from the result no
longer exist in Jira), or None when Jira could not be queried, in which case
the cached values are left untouched.
"""
keys = [row.issue_key for row in rows if row.issue_key]
if not keys:
return {}
try:
statuses = jira_integration.get_issues_status(keys)
except JiraBaseException as error:
logger.warning(
"Could not refresh Jira issue statuses, keeping cached values: %s",
error.message or error,
)
return None
except Exception:
logger.exception("Could not refresh Jira issue statuses, keeping cached values")
return None
now = timezone.now()
for row in rows:
status = statuses.get(row.issue_key)
if status is None:
# The issue is gone: keep the key for reference but mark it as done so
# the next send creates a fresh issue
row.issue_status = ""
row.issue_status_category = JiraIssue.StatusCategoryChoices.DONE
else:
row.issue_status = status.get("status", "")[:64]
row.issue_status_category = status.get("status_category", "")[:16]
row.status_synced_at = now
with rls_transaction(tenant_id):
JiraIssue.objects.bulk_update(
rows, ["issue_status", "issue_status_category", "status_synced_at"]
)
return statuses
def _reserve_jira_issue(
tenant_id: str,
integration_id: str,
provider_id: str,
finding_uid: str,
finding_id: str,
project_key: str,
) -> JiraIssue | None:
"""Claim the (integration, provider, finding uid) slot before calling Jira.
The unique constraint makes this the arbiter between concurrent runs: only
the run that inserts the row (or reclaims an expired reservation) sends the
finding. Returns None when another run owns the slot.
"""
with rls_transaction(tenant_id):
try:
row, created = JiraIssue.objects.get_or_create(
tenant_id=tenant_id,
integration_id=integration_id,
provider_id=provider_id,
finding_uid=finding_uid,
defaults={"finding_id": finding_id, "project_key": project_key},
)
except IntegrityError:
return None
if created:
return row
if row.issue_key:
# Linked by a concurrent run between the pre-check and now
return None
if timezone.now() - row.updated_at < JIRA_RESERVATION_TTL:
# Another run is sending this finding right now
return None
# Expired reservation from a run that died mid-send: reclaim it
row.finding_id = finding_id
row.project_key = project_key
row.save(update_fields=["finding_id", "project_key", "updated_at"])
return row
def _link_jira_issue(
tenant_id: str, row: JiraIssue, issue: dict, finding_id: str, project_key: str
) -> None:
"""Point the row at the issue Jira just created."""
with rls_transaction(tenant_id):
row.issue_key = (issue.get("key") or "")[:64]
row.issue_id = str(issue.get("id") or "")[:64]
row.issue_url = (issue.get("url") or "")[:2048]
row.project_key = project_key
row.finding_id = finding_id
row.issue_status = ""
row.issue_status_category = JiraIssue.StatusCategoryChoices.NEW
row.status_synced_at = None
row.save(
update_fields=[
"issue_key",
"issue_id",
"issue_url",
"project_key",
"finding_id",
"issue_status",
"issue_status_category",
"status_synced_at",
"updated_at",
]
)
def _release_jira_issue(tenant_id: str, row: JiraIssue) -> None:
"""Drop a reservation whose send failed so the finding can be retried."""
if row.issue_key:
# A previously linked (now closed) issue stays linked; the send failed so
# there is nothing newer to point at
return
with rls_transaction(tenant_id):
JiraIssue.objects.filter(id=row.id, issue_key="").delete()
def _skipped_entry(finding_id: str, row: JiraIssue) -> dict:
return {
"finding_id": str(finding_id),
"issue_key": row.issue_key,
"issue_url": row.issue_url,
"issue_status": row.issue_status,
}
def send_findings_to_jira(
tenant_id: str,
integration_id: str,
@@ -562,14 +739,50 @@ def send_findings_to_jira(
issue_type: str,
finding_ids: list[str],
):
"""Create one Jira issue per finding, skipping findings that already have one.
Findings are matched to existing issues by (integration, provider, finding
uid), so a finding that was already sent in a previous scan is recognised.
Findings whose linked issue is still open are skipped and reported; findings
whose issue is closed or was deleted in Jira get a new issue that replaces
the link.
"""
with rls_transaction(tenant_id):
integration = Integration.objects.get(id=integration_id)
jira_integration = initialize_prowler_integration(integration)
tenant_info = get_tenant_name(tenant_id)
finding_refs = _load_finding_refs(finding_ids)
existing = _load_existing_jira_issues(tenant_id, integration_id, finding_refs)
# Refresh the status of the linked issues in bulk so closed/deleted ones can be
# replaced. If Jira cannot be queried the linked findings are skipped as-is.
linked_rows = [row for row in existing.values() if row.issue_key]
statuses = (
_refresh_jira_issue_statuses(tenant_id, jira_integration, linked_rows)
if linked_rows
else {}
)
num_tickets_created = 0
error_messages = []
skipped: list[dict] = []
error_messages: list[str] = []
created_rows: list[JiraIssue] = []
for finding_id in finding_ids:
finding_id = str(finding_id)
provider_id, finding_uid = finding_refs.get(finding_id, (None, None))
row = existing.get((provider_id, finding_uid)) if provider_id else None
if row is not None:
if row.issue_key:
if statuses is None or not row.is_done:
# Still open (or status unknown): already ticketed
skipped.append(_skipped_entry(finding_id, row))
continue
# Closed or deleted in Jira: create a replacement below
elif timezone.now() - row.updated_at < JIRA_RESERVATION_TTL:
# Another run is sending this finding right now
skipped.append(_skipped_entry(finding_id, row))
continue
with rls_transaction(tenant_id):
finding_instance = (
Finding.all_objects.select_related("scan__provider")
@@ -599,69 +812,98 @@ def send_findings_to_jira(
remediation_code = remediation.get("code", {})
provider_type = finding_instance.scan.provider.provider
if provider_id is None:
provider_id = str(finding_instance.scan.provider_id)
finding_uid = finding_instance.uid
issue_labels = build_jira_issue_labels(
finding_uid=finding_instance.uid,
finding_uid=finding_uid,
provider=provider_type,
severity=finding_instance.severity,
check_id=finding_instance.check_id,
)
finding_url = build_jira_finding_url(finding_instance.uid)
finding_url = build_jira_finding_url(finding_uid)
try:
# Send the individual finding to Jira
result = jira_integration.send_finding(
check_id=finding_instance.check_id,
check_title=check_metadata.get("checktitle", ""),
severity=finding_instance.severity,
status=finding_instance.status,
status_extended=finding_instance.status_extended or "",
provider=provider_type,
region=region,
resource_uid=resource_uid,
resource_name=resource_name,
risk=check_metadata.get("risk", ""),
recommendation_text=recommendation.get("text", ""),
recommendation_url=recommendation.get("url", ""),
remediation_code_native_iac=remediation_code.get("nativeiac", ""),
remediation_code_terraform=remediation_code.get("terraform", ""),
remediation_code_cli=remediation_code.get("cli", ""),
remediation_code_other=remediation_code.get("other", ""),
resource_tags=resource_tags,
compliance=finding_instance.compliance or {},
project_key=project_key,
issue_type=issue_type,
issue_labels=issue_labels,
finding_url=finding_url,
tenant_info=tenant_info,
)
except JiraBaseException as error:
error_message = error.message or JIRA_GENERIC_SEND_ERROR
logger.exception(
"Failed to send finding %s to Jira: %s", finding_id, error_message
)
error_messages.append(error_message)
continue
except Exception:
logger.exception("Failed to send finding %s to Jira", finding_id)
error_messages.append(JIRA_GENERIC_SEND_ERROR)
if row is None:
row = _reserve_jira_issue(
tenant_id,
integration_id,
provider_id,
finding_uid,
finding_id,
project_key,
)
if row is None:
skipped.append({"finding_id": finding_id})
continue
if result:
num_tickets_created += 1
logger.info(
"Finding %s sent to Jira as %s",
finding_id,
result.get("key") if isinstance(result, dict) else result,
)
else:
error_message = JIRA_GENERIC_SEND_ERROR
logger.error(error_message)
error_messages.append(error_message)
try:
# Send the individual finding to Jira
result = jira_integration.send_finding(
check_id=finding_instance.check_id,
check_title=check_metadata.get("checktitle", ""),
severity=finding_instance.severity,
status=finding_instance.status,
status_extended=finding_instance.status_extended or "",
provider=provider_type,
region=region,
resource_uid=resource_uid,
resource_name=resource_name,
risk=check_metadata.get("risk", ""),
recommendation_text=recommendation.get("text", ""),
recommendation_url=recommendation.get("url", ""),
remediation_code_native_iac=remediation_code.get("nativeiac", ""),
remediation_code_terraform=remediation_code.get("terraform", ""),
remediation_code_cli=remediation_code.get("cli", ""),
remediation_code_other=remediation_code.get("other", ""),
resource_tags=resource_tags,
compliance=finding_instance.compliance or {},
project_key=project_key,
issue_type=issue_type,
issue_labels=issue_labels,
finding_url=finding_url,
tenant_info=tenant_info,
)
except JiraBaseException as error:
error_message = error.message or JIRA_GENERIC_SEND_ERROR
logger.exception(
"Failed to send finding %s to Jira: %s", finding_id, error_message
)
error_messages.append(error_message)
_release_jira_issue(tenant_id, row)
continue
except Exception:
logger.exception("Failed to send finding %s to Jira", finding_id)
error_messages.append(JIRA_GENERIC_SEND_ERROR)
_release_jira_issue(tenant_id, row)
continue
if result:
num_tickets_created += 1
issue = result if isinstance(result, dict) else {}
logger.info(
"Finding %s sent to Jira as %s", finding_id, issue.get("key") or result
)
_link_jira_issue(tenant_id, row, issue, finding_id, project_key)
created_rows.append(row)
else:
error_message = JIRA_GENERIC_SEND_ERROR
logger.error(error_message)
error_messages.append(error_message)
_release_jira_issue(tenant_id, row)
# Record the initial status of the issues just created, in bulk
if created_rows:
_refresh_jira_issue_statuses(
tenant_id, jira_integration, [row for row in created_rows if row.issue_key]
)
result = {
"created_count": num_tickets_created,
"failed_count": len(finding_ids) - num_tickets_created,
"skipped_count": len(skipped),
"failed_count": len(finding_ids) - num_tickets_created - len(skipped),
}
if skipped:
result["skipped"] = skipped[:JIRA_SKIPPED_REPORT_LIMIT]
if error_messages:
result["error"] = "; ".join(dict.fromkeys(error_messages))
api/src/backend/tasks/tests/test_deletion.py
+17 -1
View File
@@ -2,7 +2,7 @@ from unittest.mock import MagicMock, call, patch
import pytest
from api.attack_paths import database as graph_database
from api.models import Provider, Tenant, TenantComplianceSummary
from api.models import JiraIssue, Provider, Tenant, TenantComplianceSummary
from django.core.exceptions import ObjectDoesNotExist
from tasks.jobs.deletion import delete_provider, delete_tenant
@@ -33,6 +33,22 @@ class TestDeleteProvider:
str(instance.id),
)
def test_delete_provider_removes_jira_issues(self, jira_issues_fixture):
linked, other_provider_issue, reservation = jira_issues_fixture
provider = linked.provider
tenant_id = str(provider.tenant_id)
with (
patch("tasks.jobs.deletion.graph_database.get_database_name"),
patch("tasks.jobs.deletion.graph_database.drop_subgraph"),
):
delete_provider(tenant_id, provider.id)
remaining = set(JiraIssue.objects.values_list("id", flat=True))
assert linked.id not in remaining
assert reservation.id not in remaining
# Issues of other providers are untouched
assert other_provider_issue.id in remaining
def test_delete_provider_does_not_exist(self, tenants_fixture):
with (
patch(
api/src/backend/tasks/tests/test_integrations.py
+313 -5
View File
@@ -1,12 +1,15 @@
from datetime import UTC, datetime
import itertools
from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
import pytest
from api.db_router import READ_REPLICA_ALIAS, MainRouter
from api.models import Integration
from api.db_utils import rls_transaction
from api.models import Integration, JiraIssue
from api.utils import prowler_integration_connection_test
from django.db import OperationalError
from django.test import override_settings
from django.utils import timezone
from prowler.lib.outputs.jira.exceptions.exceptions import (
JiraRefreshTokenError,
JiraRequiredCustomFieldsError,
@@ -14,6 +17,7 @@ from prowler.lib.outputs.jira.exceptions.exceptions import (
from prowler.providers.aws.lib.security_hub.security_hub import SecurityHubConnection
from prowler.providers.common.models import Connection
from tasks.jobs.integrations import (
JIRA_RESERVATION_TTL,
build_jira_finding_url,
build_jira_issue_labels,
get_s3_client_from_integration,
@@ -1656,6 +1660,27 @@ class TestSecurityHubIntegrationUploads:
@pytest.mark.django_db
class TestJiraIntegration:
"""Sending findings to Jira, with the dedup bookkeeping stubbed out.
These tests use fake tenant ids and fully mocked findings; the dedup helpers
are exercised with real rows in TestJiraIssueDedup.
"""
@pytest.fixture(autouse=True)
def no_dedup_bookkeeping(self):
reservation = MagicMock()
reservation.issue_key = ""
with patch.multiple(
"tasks.jobs.integrations",
_load_finding_refs=MagicMock(return_value={}),
_load_existing_jira_issues=MagicMock(return_value={}),
_refresh_jira_issue_statuses=MagicMock(return_value={}),
_reserve_jira_issue=MagicMock(return_value=reservation),
_link_jira_issue=MagicMock(),
_release_jira_issue=MagicMock(),
):
yield
@patch("tasks.jobs.integrations.rls_transaction")
@patch("tasks.jobs.integrations.Finding")
@patch("tasks.jobs.integrations.Integration")
@@ -1765,7 +1790,7 @@ class TestJiraIntegration:
)
# Assertions
assert result == {"created_count": 2, "failed_count": 0}
assert result == {"created_count": 2, "skipped_count": 0, "failed_count": 0}
# Verify Jira integration was initialized
mock_initialize_integration.assert_called_once_with(integration)
@@ -1885,6 +1910,7 @@ class TestJiraIntegration:
# Assertions
assert result == {
"created_count": 2,
"skipped_count": 0,
"failed_count": 1,
"error": "Failed to create Jira issue.",
}
@@ -1951,6 +1977,7 @@ class TestJiraIntegration:
assert result == {
"created_count": 0,
"skipped_count": 0,
"failed_count": 1,
"error": error_message,
}
@@ -2019,6 +2046,7 @@ class TestJiraIntegration:
assert result == {
"created_count": 0,
"skipped_count": 0,
"failed_count": 1,
"error": error_message,
}
@@ -2083,6 +2111,7 @@ class TestJiraIntegration:
assert result == {
"created_count": 0,
"skipped_count": 0,
"failed_count": 1,
"error": "Failed to create Jira issue.",
}
@@ -2160,7 +2189,7 @@ class TestJiraIntegration:
)
# Assertions
assert result == {"created_count": 1, "failed_count": 0}
assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0}
# Verify send_finding was called with empty resource fields
call_kwargs = mock_jira_integration.send_finding.call_args.kwargs
@@ -2223,7 +2252,7 @@ class TestJiraIntegration:
)
# Assertions
assert result == {"created_count": 1, "failed_count": 0}
assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0}
# Verify send_finding was called with default/empty values
call_kwargs = mock_jira_integration.send_finding.call_args.kwargs
@@ -2314,3 +2343,282 @@ class TestJiraFindingReference:
def test_get_tenant_name_unknown_or_invalid(self):
assert get_tenant_name("00000000-0000-0000-0000-000000000000") == ""
assert get_tenant_name("not-a-uuid") == ""
@pytest.mark.django_db
class TestJiraIssueDedup:
"""send_findings_to_jira with real JiraIssue rows: skip, replace, reserve."""
@pytest.fixture
def jira_mock(self):
jira = MagicMock()
counter = itertools.count(1)
def _create_issue(**kwargs):
number = next(counter)
return {
"key": f"TEST-{number}",
"id": str(10000 + number),
"url": f"https://test.atlassian.net/browse/TEST-{number}",
}
jira.send_finding.side_effect = _create_issue
jira.get_issues_status.return_value = {}
return jira
@pytest.fixture
def send(self, jira_mock, jira_integration_fixture):
def _send(finding_ids):
with patch(
"tasks.jobs.integrations.initialize_prowler_integration",
return_value=jira_mock,
):
return send_findings_to_jira(
str(jira_integration_fixture.tenant_id),
str(jira_integration_fixture.id),
"TEST",
"Task",
[str(finding_id) for finding_id in finding_ids],
)
return _send
@staticmethod
def _rows(integration):
with rls_transaction(str(integration.tenant_id)):
return {
row.finding_uid: row
for row in JiraIssue.objects.filter(integration=integration)
}
def test_first_send_links_findings(
self, send, jira_mock, jira_integration_fixture, findings_fixture
):
finding1, finding2 = findings_fixture
jira_mock.get_issues_status.return_value = {
"TEST-1": {"id": "10001", "status": "To Do", "status_category": "new"},
"TEST-2": {"id": "10002", "status": "To Do", "status_category": "new"},
}
result = send([finding1.id, finding2.id])
assert result == {"created_count": 2, "skipped_count": 0, "failed_count": 0}
assert jira_mock.send_finding.call_count == 2
rows = self._rows(jira_integration_fixture)
assert set(rows) == {finding1.uid, finding2.uid}
row = rows[finding1.uid]
assert row.issue_key == "TEST-1"
assert row.issue_id == "10001"
assert row.issue_url == "https://test.atlassian.net/browse/TEST-1"
assert row.project_key == "TEST"
assert row.finding_id == finding1.id
assert row.provider_id == finding1.scan.provider_id
# Status of the new issues is fetched once, in bulk, after creation
jira_mock.get_issues_status.assert_called_once_with(["TEST-1", "TEST-2"])
assert row.issue_status == "To Do"
assert row.issue_status_category == "new"
assert row.status_synced_at is not None
def test_second_send_skips_open_issue(
self, send, jira_mock, jira_integration_fixture, findings_fixture
):
finding1, _ = findings_fixture
send([finding1.id])
jira_mock.send_finding.reset_mock()
jira_mock.get_issues_status.reset_mock()
jira_mock.get_issues_status.return_value = {
"TEST-1": {
"id": "10001",
"status": "In Progress",
"status_category": "indeterminate",
}
}
result = send([finding1.id])
assert result == {
"created_count": 0,
"skipped_count": 1,
"failed_count": 0,
"skipped": [
{
"finding_id": str(finding1.id),
"issue_key": "TEST-1",
"issue_url": "https://test.atlassian.net/browse/TEST-1",
"issue_status": "In Progress",
}
],
}
jira_mock.send_finding.assert_not_called()
jira_mock.get_issues_status.assert_called_once_with(["TEST-1"])
row = self._rows(jira_integration_fixture)[finding1.uid]
assert row.issue_key == "TEST-1"
# The refreshed status is cached on the row
assert row.issue_status == "In Progress"
assert row.issue_status_category == "indeterminate"
@pytest.mark.parametrize(
"status_response",
[
{"TEST-1": {"id": "10001", "status": "Done", "status_category": "done"}},
{}, # deleted in Jira
],
ids=["closed", "deleted"],
)
def test_closed_or_deleted_issue_is_replaced(
self,
send,
jira_mock,
jira_integration_fixture,
findings_fixture,
status_response,
):
finding1, _ = findings_fixture
send([finding1.id])
jira_mock.send_finding.reset_mock()
jira_mock.get_issues_status.return_value = status_response
result = send([finding1.id])
assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0}
jira_mock.send_finding.assert_called_once()
rows = self._rows(jira_integration_fixture)
assert len(rows) == 1
assert rows[finding1.uid].issue_key == "TEST-2"
assert (
rows[finding1.uid].issue_url == "https://test.atlassian.net/browse/TEST-2"
)
def test_status_lookup_failure_skips_linked_findings(
self, send, jira_mock, jira_integration_fixture, findings_fixture
):
finding1, _ = findings_fixture
send([finding1.id])
jira_mock.send_finding.reset_mock()
jira_mock.get_issues_status.side_effect = JiraRefreshTokenError(
message="token expired"
)
result = send([finding1.id])
assert result["created_count"] == 0
assert result["skipped_count"] == 1
jira_mock.send_finding.assert_not_called()
# Cached status untouched
row = self._rows(jira_integration_fixture)[finding1.uid]
assert row.issue_key == "TEST-1"
def test_failed_send_releases_reservation(
self, send, jira_mock, jira_integration_fixture, findings_fixture
):
finding1, _ = findings_fixture
jira_mock.send_finding.side_effect = JiraRequiredCustomFieldsError(
message="custom fields"
)
result = send([finding1.id])
assert result["created_count"] == 0
assert result["failed_count"] == 1
assert result["error"] == "custom fields"
assert self._rows(jira_integration_fixture) == {}
def test_failed_replacement_keeps_previous_link(
self, send, jira_mock, jira_integration_fixture, findings_fixture
):
finding1, _ = findings_fixture
send([finding1.id])
jira_mock.get_issues_status.return_value = {
"TEST-1": {"id": "10001", "status": "Done", "status_category": "done"}
}
jira_mock.send_finding.side_effect = Exception("boom")
result = send([finding1.id])
assert result["failed_count"] == 1
row = self._rows(jira_integration_fixture)[finding1.uid]
assert row.issue_key == "TEST-1"
assert row.issue_status_category == "done"
def test_fresh_reservation_is_skipped_and_stale_one_reclaimed(
self, send, jira_mock, jira_integration_fixture, findings_fixture
):
finding1, _ = findings_fixture
tenant_id = str(jira_integration_fixture.tenant_id)
with rls_transaction(tenant_id):
reservation = JiraIssue.objects.create(
tenant_id=tenant_id,
integration=jira_integration_fixture,
provider_id=finding1.scan.provider_id,
finding_uid=finding1.uid,
finding_id=finding1.id,
project_key="TEST",
)
# Another run is sending this finding right now
result = send([finding1.id])
assert result == {
"created_count": 0,
"skipped_count": 1,
"failed_count": 0,
"skipped": [
{
"finding_id": str(finding1.id),
"issue_key": "",
"issue_url": "",
"issue_status": "",
}
],
}
jira_mock.send_finding.assert_not_called()
# The run died: the reservation expired and is reclaimed
with rls_transaction(tenant_id):
JiraIssue.objects.filter(id=reservation.id).update(
updated_at=timezone.now() - JIRA_RESERVATION_TTL - timedelta(minutes=1)
)
result = send([finding1.id])
assert result == {"created_count": 1, "skipped_count": 0, "failed_count": 0}
rows = self._rows(jira_integration_fixture)
assert len(rows) == 1
assert rows[finding1.uid].id == reservation.id
assert rows[finding1.uid].issue_key == "TEST-1"
def test_other_integration_does_not_dedup(
self,
send,
jira_mock,
jira_integration_fixture,
findings_fixture,
tenants_fixture,
):
finding1, _ = findings_fixture
tenant_id = str(jira_integration_fixture.tenant_id)
with rls_transaction(tenant_id):
other = Integration.objects.create(
tenant_id=tenant_id,
enabled=True,
connected=True,
integration_type=Integration.IntegrationChoices.JIRA,
configuration={"projects": {"OTHER": "Other"}},
credentials={
"domain": "other",
"user_mail": "a@b.com",
"api_token": "t",
},
)
send([finding1.id])
jira_mock.send_finding.reset_mock()
with patch(
"tasks.jobs.integrations.initialize_prowler_integration",
return_value=jira_mock,
):
result = send_findings_to_jira(
tenant_id, str(other.id), "OTHER", "Task", [str(finding1.id)]
)
assert result["created_count"] == 1
jira_mock.send_finding.assert_called_once()
with rls_transaction(tenant_id):
assert JiraIssue.objects.filter(finding_uid=finding1.uid).count() == 2
docs/user-guide/tutorials/prowler-app-jira-integration.mdx
+9
View File
@@ -134,6 +134,15 @@ Every Jira issue created from a single Finding carries a stable reference back t
Prowler Cloud always includes the Finding URL. In a self-hosted Prowler App, set `DJANGO_UI_BASE_URL` in the API environment (for example, `https://prowler.example.com`) to enable it. When the variable is empty, the issue is created without the link.
### Sending a Finding That Already Has a Jira Issue
Prowler remembers the Jira issue created for each Finding, keyed by the Finding UID, so sending the same Finding again does not create a duplicate issue:
* If the linked issue is still open in Jira, the Finding is skipped and the existing issue key is reported in the task result (`skipped_count`, `skipped`).
* If the linked issue is closed (any Jira status in the **Done** category) or was deleted, a new issue is created and becomes the linked issue for that Finding.
The link, and the last status observed in Jira, are available through the API at `GET /api/v1/jira-issues` (filter by `finding_uid`, `finding_uid__in`, `provider_id`, `integration` or `issue_key`). Each Jira integration keeps its own links, so the same Finding can have one issue per integration.
## Integration Status
Monitor and manage your Jira integrations through the management interface: