From 10a4c289220290d9c8363e5b84c5decc3e8afb80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Jes=C3=BAs=20Pe=C3=B1a=20Rodr=C3=ADguez?= Date: Wed, 15 Jan 2025 11:25:41 +0100 Subject: [PATCH] feat(finding): add first_seen attribute (#6460) --- .../api/fixtures/dev/5_dev_findings.json | 39 +++++++++++++++++++ .../migrations/0006_findings_first_seen.py | 15 +++++++ api/src/backend/api/models.py | 1 + api/src/backend/api/specs/v1.yaml | 7 ++++ api/src/backend/api/v1/serializers.py | 1 + api/src/backend/conftest.py | 2 + api/src/backend/tasks/jobs/scan.py | 21 ++++++---- 7 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 api/src/backend/api/migrations/0006_findings_first_seen.py diff --git a/api/src/backend/api/fixtures/dev/5_dev_findings.json b/api/src/backend/api/fixtures/dev/5_dev_findings.json index 8a02b6c2ed..acffd12c3e 100644 --- a/api/src/backend/api/fixtures/dev/5_dev_findings.json +++ b/api/src/backend/api/fixtures/dev/5_dev_findings.json @@ -6,6 +6,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.823Z", "updated_at": "2024-10-18T10:46:04.841Z", + "first_seen_at": "2024-10-18T10:46:04.823Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-south-2-112233445566", "delta": "new", "status": "FAIL", @@ -61,6 +62,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.855Z", "updated_at": "2024-10-18T10:46:04.858Z", + "first_seen_at": "2024-10-18T10:46:04.855Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-west-3-112233445566", "delta": "new", "status": "FAIL", @@ -116,6 +118,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.869Z", "updated_at": "2024-10-18T10:46:04.876Z", + "first_seen_at": "2024-10-18T10:46:04.869Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-central-2-112233445566", "delta": "new", "status": "FAIL", @@ -171,6 +174,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.888Z", "updated_at": "2024-10-18T10:46:04.892Z", + "first_seen_at": "2024-10-18T10:46:04.888Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-west-1-112233445566", "delta": "new", "status": "FAIL", @@ -226,6 +230,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.901Z", "updated_at": "2024-10-18T10:46:04.905Z", + "first_seen_at": "2024-10-18T10:46:04.901Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-east-2-112233445566", "delta": "new", "status": "FAIL", @@ -281,6 +286,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.915Z", "updated_at": "2024-10-18T10:46:04.919Z", + "first_seen_at": "2024-10-18T10:46:04.915Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-south-1-112233445566", "delta": "new", "status": "FAIL", @@ -336,6 +342,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.929Z", "updated_at": "2024-10-18T10:46:04.934Z", + "first_seen_at": "2024-10-18T10:46:04.929Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-west-1-112233445566", "delta": "new", "status": "FAIL", @@ -391,6 +398,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.944Z", "updated_at": "2024-10-18T10:46:04.947Z", + "first_seen_at": "2024-10-18T10:46:04.944Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ca-central-1-112233445566", "delta": "new", "status": "FAIL", @@ -446,6 +454,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.957Z", "updated_at": "2024-10-18T10:46:04.962Z", + "first_seen_at": "2024-10-18T10:46:04.957Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-east-1-ConsoleAnalyzer-83b66ad7-d024-454e-b851-52d11cc1cf7c", "delta": "new", "status": "PASS", @@ -501,6 +510,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.971Z", "updated_at": "2024-10-18T10:46:04.975Z", + "first_seen_at": "2024-10-18T10:46:04.971Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-west-2-112233445566", "delta": "new", "status": "FAIL", @@ -556,6 +566,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.984Z", "updated_at": "2024-10-18T10:46:04.989Z", + "first_seen_at": "2024-10-18T10:46:04.984Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-sa-east-1-112233445566", "delta": "new", "status": "FAIL", @@ -611,6 +622,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:04.999Z", "updated_at": "2024-10-18T10:46:05.003Z", + "first_seen_at": "2024-10-18T10:46:04.999Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-north-1-112233445566", "delta": "new", "status": "FAIL", @@ -666,6 +678,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.013Z", "updated_at": "2024-10-18T10:46:05.018Z", + "first_seen_at": "2024-10-18T10:46:05.013Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-west-2-112233445566", "delta": "new", "status": "FAIL", @@ -721,6 +734,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.029Z", "updated_at": "2024-10-18T10:46:05.033Z", + "first_seen_at": "2024-10-18T10:46:05.029Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-southeast-1-112233445566", "delta": "new", "status": "FAIL", @@ -776,6 +790,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.045Z", "updated_at": "2024-10-18T10:46:05.050Z", + "first_seen_at": "2024-10-18T10:46:05.045Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-central-1-112233445566", "delta": "new", "status": "FAIL", @@ -831,6 +846,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.061Z", "updated_at": "2024-10-18T10:46:05.065Z", + "first_seen_at": "2024-10-18T10:46:05.061Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-northeast-1-112233445566", "delta": "new", "status": "FAIL", @@ -886,6 +902,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.080Z", "updated_at": "2024-10-18T10:46:05.085Z", + "first_seen_at": "2024-10-18T10:46:05.080Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-southeast-2-112233445566", "delta": "new", "status": "FAIL", @@ -941,6 +958,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.099Z", "updated_at": "2024-10-18T10:46:05.104Z", + "first_seen_at": "2024-10-18T10:46:05.099Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-northeast-2-112233445566", "delta": "new", "status": "FAIL", @@ -996,6 +1014,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T10:46:05.115Z", "updated_at": "2024-10-18T10:46:05.121Z", + "first_seen_at": "2024-10-18T10:46:05.115Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-northeast-3-112233445566", "delta": "new", "status": "FAIL", @@ -1051,6 +1070,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.489Z", "updated_at": "2024-10-18T11:16:24.506Z", + "first_seen_at": "2024-10-18T10:46:04.823Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-south-2-112233445566", "delta": null, "status": "FAIL", @@ -1106,6 +1126,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.518Z", "updated_at": "2024-10-18T11:16:24.521Z", + "first_seen_at": "2024-10-18T10:46:04.855Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-west-3-112233445566", "delta": null, "status": "FAIL", @@ -1161,6 +1182,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.526Z", "updated_at": "2024-10-18T11:16:24.529Z", + "first_seen_at": "2024-10-18T10:46:04.869Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-central-2-112233445566", "delta": null, "status": "FAIL", @@ -1216,6 +1238,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.535Z", "updated_at": "2024-10-18T11:16:24.538Z", + "first_seen_at": "2024-10-18T10:46:04.888Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-west-1-112233445566", "delta": null, "status": "FAIL", @@ -1271,6 +1294,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.544Z", "updated_at": "2024-10-18T11:16:24.546Z", + "first_seen_at": "2024-10-18T10:46:04.901Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-east-2-112233445566", "delta": null, "status": "FAIL", @@ -1326,6 +1350,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.551Z", "updated_at": "2024-10-18T11:16:24.554Z", + "first_seen_at": "2024-10-18T10:46:04.915Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-south-1-112233445566", "delta": null, "status": "FAIL", @@ -1381,6 +1406,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.560Z", "updated_at": "2024-10-18T11:16:24.562Z", + "first_seen_at": "2024-10-18T10:46:04.929Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-west-1-112233445566", "delta": null, "status": "FAIL", @@ -1436,6 +1462,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.567Z", "updated_at": "2024-10-18T11:16:24.569Z", + "first_seen_at": "2024-10-18T10:46:04.944Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ca-central-1-112233445566", "delta": null, "status": "FAIL", @@ -1491,6 +1518,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.573Z", "updated_at": "2024-10-18T11:16:24.575Z", + "first_seen_at": "2024-10-18T10:46:04.957Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-east-1-ConsoleAnalyzer-83b66ad7-d024-454e-b851-52d11cc1cf7c", "delta": null, "status": "PASS", @@ -1546,6 +1574,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.580Z", "updated_at": "2024-10-18T11:16:24.582Z", + "first_seen_at": "2024-10-18T10:46:04.971Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-west-2-112233445566", "delta": null, "status": "FAIL", @@ -1601,6 +1630,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.587Z", "updated_at": "2024-10-18T11:16:24.589Z", + "first_seen_at": "2024-10-18T10:46:04.984Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-sa-east-1-112233445566", "delta": null, "status": "FAIL", @@ -1656,6 +1686,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.595Z", "updated_at": "2024-10-18T11:16:24.597Z", + "first_seen_at": "2024-10-18T10:46:04.999Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-north-1-112233445566", "delta": null, "status": "FAIL", @@ -1711,6 +1742,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.602Z", "updated_at": "2024-10-18T11:16:24.604Z", + "first_seen_at": "2024-10-18T10:46:05.013Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-us-west-2-112233445566", "delta": null, "status": "FAIL", @@ -1766,6 +1798,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.610Z", "updated_at": "2024-10-18T11:16:24.612Z", + "first_seen_at": "2024-10-18T10:46:05.029Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-southeast-1-112233445566", "delta": null, "status": "FAIL", @@ -1821,6 +1854,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.617Z", "updated_at": "2024-10-18T11:16:24.620Z", + "first_seen_at": "2024-10-18T10:46:05.045Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-eu-central-1-112233445566", "delta": null, "status": "FAIL", @@ -1876,6 +1910,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.625Z", "updated_at": "2024-10-18T11:16:24.627Z", + "first_seen_at": "2024-10-18T10:46:05.061Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-northeast-1-112233445566", "delta": null, "status": "FAIL", @@ -1931,6 +1966,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.632Z", "updated_at": "2024-10-18T11:16:24.634Z", + "first_seen_at": "2024-10-18T10:46:05.080Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-southeast-2-112233445566", "delta": null, "status": "FAIL", @@ -1986,6 +2022,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.639Z", "updated_at": "2024-10-18T11:16:24.642Z", + "first_seen_at": "2024-10-18T10:46:05.099Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-northeast-2-112233445566", "delta": null, "status": "FAIL", @@ -2041,6 +2078,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:24.646Z", "updated_at": "2024-10-18T11:16:24.648Z", + "first_seen_at": "2024-10-18T10:46:05.115Z", "uid": "prowler-aws-accessanalyzer_enabled-112233445566-ap-northeast-3-112233445566", "delta": null, "status": "FAIL", @@ -2096,6 +2134,7 @@ "tenant": "12646005-9067-4d2a-a098-8bb378604362", "inserted_at": "2024-10-18T11:16:26.033Z", "updated_at": "2024-10-18T11:16:26.045Z", + "first_seen_at": "2024-10-18T11:16:26.033Z", "uid": "prowler-aws-account_security_contact_information_is_registered-112233445566-us-east-1-112233445566", "delta": "new", "status": "MANUAL", diff --git a/api/src/backend/api/migrations/0006_findings_first_seen.py b/api/src/backend/api/migrations/0006_findings_first_seen.py new file mode 100644 index 0000000000..e457ffa814 --- /dev/null +++ b/api/src/backend/api/migrations/0006_findings_first_seen.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0005_rbac_missing_admin_roles"), + ] + + operations = [ + migrations.AddField( + model_name="finding", + name="first_seen_at", + field=models.DateTimeField(editable=False, null=True), + ), + ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index 3c240a2f7c..d1b9434246 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -615,6 +615,7 @@ class Finding(PostgresPartitionedModel, RowLevelSecurityProtectedModel): id = models.UUIDField(primary_key=True, default=uuid7, editable=False) inserted_at = models.DateTimeField(auto_now_add=True, editable=False) updated_at = models.DateTimeField(auto_now=True, editable=False) + first_seen_at = models.DateTimeField(editable=False, null=True) uid = models.CharField(max_length=300) delta = FindingDeltaEnumField( diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 1df691583f..7dc3af2104 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -257,6 +257,7 @@ paths: - raw_result - inserted_at - updated_at + - first_seen_at - url - scan - resources @@ -760,6 +761,7 @@ paths: - raw_result - inserted_at - updated_at + - first_seen_at - url - scan - resources @@ -5949,6 +5951,11 @@ components: type: string format: date-time readOnly: true + first_seen_at: + type: string + format: date-time + readOnly: true + nullable: true required: - uid - status diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 8405b1dfdc..e73825daa1 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -905,6 +905,7 @@ class FindingSerializer(RLSSerializer): "raw_result", "inserted_at", "updated_at", + "first_seen_at", "url", # Relationships "scan", diff --git a/api/src/backend/conftest.py b/api/src/backend/conftest.py index 014e660480..04dbd19753 100644 --- a/api/src/backend/conftest.py +++ b/api/src/backend/conftest.py @@ -626,6 +626,7 @@ def findings_fixture(scans_fixture, resources_fixture): "CheckId": "test_check_id", "Description": "test description apple sauce", }, + first_seen_at="2024-01-02T00:00:00Z", ) finding1.add_resources([resource1]) @@ -651,6 +652,7 @@ def findings_fixture(scans_fixture, resources_fixture): "CheckId": "test_check_id", "Description": "test description orange juice", }, + first_seen_at="2024-01-02T00:00:00Z", ) finding2.add_resources([resource2]) diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 41ebec6f2b..25993f5bbc 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -219,24 +219,28 @@ def perform_prowler_scan( # Process finding with rls_transaction(tenant_id): finding_uid = finding.uid + last_first_seen_at = None if finding_uid not in last_status_cache: most_recent_finding = ( Finding.objects.filter(uid=finding_uid) .order_by("-id") - .values("status") + .values("status", "first_seen_at") .first() ) - last_status = ( - most_recent_finding["status"] - if most_recent_finding - else None - ) - last_status_cache[finding_uid] = last_status + last_status = None + if most_recent_finding: + last_status = most_recent_finding["status"] + last_first_seen_at = most_recent_finding["first_seen_at"] + last_status_cache[finding_uid] = last_status, last_first_seen_at else: - last_status = last_status_cache[finding_uid] + last_status, last_first_seen_at = last_status_cache[finding_uid] status = FindingStatus[finding.status] delta = _create_finding_delta(last_status, status) + # For the findings prior to the change, when a first finding is found with delta!="new" it will be assigned a current date as first_seen_at and the successive findings with the same UID will always get the date of the previous finding. + # For new findings, when a finding (delta="new") is found for the first time, the first_seen_at attribute will be assigned the current date, the following findings will get that date. + if not last_first_seen_at: + last_first_seen_at = datetime.now(tz=timezone.utc) # Create the finding finding_instance = Finding.objects.create( @@ -251,6 +255,7 @@ def perform_prowler_scan( raw_result=finding.raw, check_id=finding.check_id, scan=scan_instance, + first_seen_at=last_first_seen_at, ) finding_instance.add_resources([resource_instance])