feat(slack): add more information about critical findings (#5042)

2026-07-24 13:01:56 +00:00 · 2024-09-19 11:02:09 -07:00
parent 7db0746416
commit 0974c5f333
4 changed files with 243 additions and 7 deletions
@@ -82,9 +82,14 @@ def extract_findings_statistics(findings: list) -> dict:
    extract_findings_statistics takes a list of findings and returns the following dict with the aggregated statistics
    {
        "total_pass": 0,
+        "total_muted_pass": 0,
        "total_fail": 0,
+        "total_muted_fail": 0,
        "resources_count": 0,
        "findings_count": 0,
+        "critical_failed_findings": [],
+        "critical_passed_findings": []
+        "all_fails_are_muted": False
    }
    """
    logger.info("Extracting audit statistics...")
@@ -96,18 +101,42 @@ def extract_findings_statistics(findings: list) -> dict:
    resources = set()
    findings_count = 0
    all_fails_are_muted = True
+    critical_severity_pass = 0
+    critical_severity_fail = 0
+    high_severity_pass = 0
+    high_severity_fail = 0
+    medium_severity_pass = 0
+    medium_severity_fail = 0
+    low_severity_pass = 0
+    low_severity_fail = 0

    for finding in findings:
        # Save the resource_id
        resources.add(finding.resource_id)

        if finding.status == "PASS":
+            if finding.check_metadata.Severity == "critical":
+                critical_severity_pass += 1
+            if finding.check_metadata.Severity == "high":
+                high_severity_pass += 1
+            if finding.check_metadata.Severity == "medium":
+                medium_severity_pass += 1
+            if finding.check_metadata.Severity == "low":
+                low_severity_pass += 1
            total_pass += 1
            findings_count += 1
            if finding.muted is True:
                muted_pass += 1

        if finding.status == "FAIL":
+            if finding.check_metadata.Severity == "critical":
+                critical_severity_fail += 1
+            if finding.check_metadata.Severity == "high":
+                high_severity_fail += 1
+            if finding.check_metadata.Severity == "medium":
+                medium_severity_fail += 1
+            if finding.check_metadata.Severity == "low":
+                low_severity_fail += 1
            total_fail += 1
            findings_count += 1
            if finding.muted is True:
@@ -121,6 +150,14 @@ def extract_findings_statistics(findings: list) -> dict:
    stats["total_muted_fail"] = muted_fail
    stats["resources_count"] = len(resources)
    stats["findings_count"] = findings_count
+    stats["total_critical_severity_fail"] = critical_severity_fail
+    stats["total_critical_severity_pass"] = critical_severity_pass
+    stats["total_high_severity_fail"] = high_severity_fail
+    stats["total_high_severity_pass"] = high_severity_pass
+    stats["total_medium_severity_fail"] = medium_severity_fail
+    stats["total_medium_severity_pass"] = medium_severity_pass
+    stats["total_low_severity_fail"] = medium_severity_fail
+    stats["total_low_severity_pass"] = medium_severity_pass
    stats["all_fails_are_muted"] = all_fails_are_muted

    return stats
@@ -121,6 +121,19 @@ class Slack:
                        "text": f"\n:white_check_mark: *{stats['total_pass']} Passed findings* ({round(stats['total_pass'] / stats['findings_count'] * 100 , 2)}%)\n",
                    },
                },
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": (
+                            "*Severities:*\n"
+                            f"• *Critical:* {stats['total_critical_severity_pass']} "
+                            f"• *High:* {stats['total_high_severity_pass']} "
+                            f"• *Medium:* {stats['total_medium_severity_pass']} "
+                            f"• *Low:* {stats['total_low_severity_pass']}"
+                        ),
+                    },
+                },
                {
                    "type": "section",
                    "text": {
@@ -128,6 +141,19 @@ class Slack:
                        "text": f"\n:x: *{stats['total_fail']} Failed findings* ({round(stats['total_fail'] / stats['findings_count'] * 100 , 2)}%)\n ",
                    },
                },
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": (
+                            "*Severities:*\n"
+                            f"• *Critical:* {stats['total_critical_severity_fail']} "
+                            f"• *High:* {stats['total_high_severity_fail']} "
+                            f"• *Medium:* {stats['total_medium_severity_fail']} "
+                            f"• *Low:* {stats['total_low_severity_fail']}"
+                        ),
+                    },
+                },
                {
                    "type": "section",
                    "text": {
@@ -299,6 +299,14 @@ class TestOutputs:
        assert stats["total_fail"] == 0
        assert stats["total_muted_pass"] == 0
        assert stats["total_muted_fail"] == 0
+        assert stats["total_critical_severity_fail"] == 0
+        assert stats["total_critical_severity_pass"] == 0
+        assert stats["total_high_severity_fail"] == 0
+        assert stats["total_high_severity_pass"] == 0
+        assert stats["total_medium_severity_fail"] == 0
+        assert stats["total_medium_severity_pass"] == 0
+        assert stats["total_low_severity_fail"] == 0
+        assert stats["total_low_severity_pass"] == 0
        assert stats["resources_count"] == 0
        assert stats["findings_count"] == 0

@@ -307,15 +315,31 @@ class TestOutputs:
        finding_1.status = "FAIL"
        finding_1.muted = True
        finding_1.resource_id = "test_resource_1"
-        findings = [finding_1]
+        finding_1.check_metadata.Severity = "medium"
+        finding_1.check_metadata.CheckID = "glue_etl_jobs_amazon_s3_encryption_enabled"
+        finding_2 = mock.MagicMock()
+        finding_2.status = "FAIL"
+        finding_2.muted = True
+        finding_2.resource_id = "test_resource_2"
+        finding_2.check_metadata.Severity = "low"
+        finding_2.check_metadata.CheckID = "lightsail_static_ip_unused"
+        findings = [finding_1, finding_2]

        stats = extract_findings_statistics(findings)
        assert stats["total_pass"] == 0
-        assert stats["total_fail"] == 1
+        assert stats["total_fail"] == 2
        assert stats["total_muted_pass"] == 0
-        assert stats["total_muted_fail"] == 1
-        assert stats["resources_count"] == 1
-        assert stats["findings_count"] == 1
+        assert stats["total_muted_fail"] == 2
+        assert stats["resources_count"] == 2
+        assert stats["findings_count"] == 2
+        assert stats["total_critical_severity_fail"] == 0
+        assert stats["total_critical_severity_pass"] == 0
+        assert stats["total_high_severity_fail"] == 0
+        assert stats["total_high_severity_pass"] == 0
+        assert stats["total_medium_severity_fail"] == 1
+        assert stats["total_medium_severity_pass"] == 0
+        assert stats["total_low_severity_fail"] == 1
+        assert stats["total_low_severity_pass"] == 0
        assert stats["all_fails_are_muted"]

    def test_extract_findings_statistics_all_fail_are_not_muted(self):
@@ -323,10 +347,16 @@ class TestOutputs:
        finding_1.status = "FAIL"
        finding_1.muted = True
        finding_1.resource_id = "test_resource_1"
+        finding_1.check_metadata.Severity = "critical"
+        finding_1.check_metadata.CheckID = "rds_instance_certificate_expiration"
        finding_2 = mock.MagicMock()
        finding_2.status = "FAIL"
        finding_2.muted = False
        finding_2.resource_id = "test_resource_1"
+        finding_2.check_metadata.Severity = "critical"
+        finding_2.check_metadata.CheckID = (
+            "autoscaling_find_secrets_ec2_launch_configuration"
+        )
        findings = [finding_1, finding_2]

        stats = extract_findings_statistics(findings)
@@ -335,6 +365,14 @@ class TestOutputs:
        assert stats["total_muted_pass"] == 0
        assert stats["total_muted_fail"] == 1
        assert stats["resources_count"] == 1
+        assert stats["total_critical_severity_fail"] == 2
+        assert stats["total_critical_severity_pass"] == 0
+        assert stats["total_high_severity_fail"] == 0
+        assert stats["total_high_severity_pass"] == 0
+        assert stats["total_medium_severity_fail"] == 0
+        assert stats["total_medium_severity_pass"] == 0
+        assert stats["total_low_severity_fail"] == 0
+        assert stats["total_low_severity_pass"] == 0
        assert stats["findings_count"] == 2
        assert not stats["all_fails_are_muted"]

@@ -343,10 +381,17 @@ class TestOutputs:
        finding_1.status = "PASS"
        finding_1.muted = True
        finding_1.resource_id = "test_resource_1"
+        finding_1.check_metadata.Severity = "critical"
+        finding_1.check_metadata.CheckID = (
+            "autoscaling_find_secrets_ec2_launch_configuration"
+        )
+
        finding_2 = mock.MagicMock()
        finding_2.status = "PASS"
        finding_2.muted = False
        finding_2.resource_id = "test_resource_1"
+        finding_2.check_metadata.Severity = "high"
+        finding_2.check_metadata.CheckID = "acm_certificates_expiration_check"
        findings = [finding_1, finding_2]

        stats = extract_findings_statistics(findings)
@@ -355,12 +400,22 @@ class TestOutputs:
        assert stats["total_muted_pass"] == 1
        assert stats["total_muted_fail"] == 0
        assert stats["resources_count"] == 1
+        assert stats["total_critical_severity_fail"] == 0
+        assert stats["total_critical_severity_pass"] == 1
+        assert stats["total_high_severity_fail"] == 0
+        assert stats["total_high_severity_pass"] == 1
+        assert stats["total_medium_severity_fail"] == 0
+        assert stats["total_medium_severity_pass"] == 0
+        assert stats["total_low_severity_fail"] == 0
+        assert stats["total_low_severity_pass"] == 0
        assert stats["findings_count"] == 2

    def test_extract_findings_statistics_all_passes_are_muted(self):
        finding_1 = mock.MagicMock()
        finding_1.status = "PASS"
        finding_1.muted = True
+        finding_1.check_metadata.Severity = "critical"
+        finding_1.check_metadata.CheckID = "rds_instance_certificate_expiration"
        finding_1.resource_id = "test_resource_1"
        findings = [finding_1]

@@ -370,6 +425,14 @@ class TestOutputs:
        assert stats["total_muted_pass"] == 1
        assert stats["total_muted_fail"] == 0
        assert stats["resources_count"] == 1
+        assert stats["total_critical_severity_fail"] == 0
+        assert stats["total_critical_severity_pass"] == 1
+        assert stats["total_high_severity_fail"] == 0
+        assert stats["total_high_severity_pass"] == 0
+        assert stats["total_medium_severity_fail"] == 0
+        assert stats["total_medium_severity_pass"] == 0
+        assert stats["total_low_severity_fail"] == 0
+        assert stats["total_low_severity_pass"] == 0
        assert stats["findings_count"] == 1

    def test_report_with_aws_provider_not_muted_pass(self):
@@ -51,6 +51,14 @@ class TestSlackIntegration:
        stats = {}
        stats["total_pass"] = 12
        stats["total_fail"] = 10
+        stats["total_critical_severity_pass"] = 4
+        stats["total_critical_severity_fail"] = 4
+        stats["total_high_severity_fail"] = 1
+        stats["total_high_severity_pass"] = 1
+        stats["total_medium_severity_fail"] = 2
+        stats["total_medium_severity_pass"] = 1
+        stats["total_low_severity_fail"] = 3
+        stats["total_low_severity_pass"] = 3
        stats["resources_count"] = 20
        stats["findings_count"] = 22

@@ -70,6 +78,14 @@ class TestSlackIntegration:
        stats = {}
        stats["total_pass"] = 12
        stats["total_fail"] = 10
+        stats["total_critical_severity_pass"] = 2
+        stats["total_critical_severity_fail"] = 4
+        stats["total_high_severity_fail"] = 1
+        stats["total_high_severity_pass"] = 1
+        stats["total_medium_severity_fail"] = 2
+        stats["total_medium_severity_pass"] = 1
+        stats["total_low_severity_fail"] = 2
+        stats["total_low_severity_pass"] = 3
        stats["resources_count"] = 20
        stats["findings_count"] = 22

@@ -96,6 +112,19 @@ class TestSlackIntegration:
                    "text": f"\n:white_check_mark: *{stats['total_pass']} Passed findings* ({round(stats['total_pass'] / stats['findings_count'] * 100 , 2)}%)\n",
                },
            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": (
+                        "*Severities:*\n"
+                        "• *Critical:* 2 "
+                        "• *High:* 1 "
+                        "• *Medium:* 1 "
+                        "• *Low:* 3"
+                    ),
+                },
+            },
            {
                "type": "section",
                "text": {
@@ -103,6 +132,19 @@ class TestSlackIntegration:
                    "text": f"\n:x: *{stats['total_fail']} Failed findings* ({round(stats['total_fail'] / stats['findings_count'] * 100 , 2)}%)\n ",
                },
            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": (
+                        "*Severities:*\n"
+                        "• *Critical:* 4 "
+                        "• *High:* 1 "
+                        "• *Medium:* 2 "
+                        "• *Low:* 2"
+                    ),
+                },
+            },
            {
                "type": "section",
                "text": {
@@ -157,12 +199,20 @@ class TestSlackIntegration:
        ]

    def test_create_message_blocks_azure(self):
-        aws_provider = set_mocked_aws_provider()
+        aws_provider = set_mocked_azure_provider()
        slack = Slack(SLACK_TOKEN, SLACK_CHANNEL, aws_provider)
        args = "--slack"
        stats = {}
        stats["total_pass"] = 12
        stats["total_fail"] = 10
+        stats["total_critical_severity_pass"] = 2
+        stats["total_critical_severity_fail"] = 4
+        stats["total_high_severity_fail"] = 1
+        stats["total_high_severity_pass"] = 1
+        stats["total_medium_severity_fail"] = 2
+        stats["total_medium_severity_pass"] = 1
+        stats["total_low_severity_fail"] = 2
+        stats["total_low_severity_pass"] = 3
        stats["resources_count"] = 20
        stats["findings_count"] = 22

@@ -191,6 +241,19 @@ class TestSlackIntegration:
                    "text": f"\n:white_check_mark: *{stats['total_pass']} Passed findings* ({round(stats['total_pass'] / stats['findings_count'] * 100 , 2)}%)\n",
                },
            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": (
+                        "*Severities:*\n"
+                        "• *Critical:* 2 "
+                        "• *High:* 1 "
+                        "• *Medium:* 1 "
+                        "• *Low:* 3"
+                    ),
+                },
+            },
            {
                "type": "section",
                "text": {
@@ -198,6 +261,19 @@ class TestSlackIntegration:
                    "text": f"\n:x: *{stats['total_fail']} Failed findings* ({round(stats['total_fail'] / stats['findings_count'] * 100 , 2)}%)\n ",
                },
            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": (
+                        "*Severities:*\n"
+                        "• *Critical:* 4 "
+                        "• *High:* 1 "
+                        "• *Medium:* 2 "
+                        "• *Low:* 2"
+                    ),
+                },
+            },
            {
                "type": "section",
                "text": {
@@ -252,12 +328,20 @@ class TestSlackIntegration:
        ]

    def test_create_message_blocks_gcp(self):
-        aws_provider = set_mocked_aws_provider()
+        aws_provider = set_mocked_gcp_provider()
        slack = Slack(SLACK_TOKEN, SLACK_CHANNEL, aws_provider)
        args = "--slack"
        stats = {}
        stats["total_pass"] = 12
        stats["total_fail"] = 10
+        stats["total_critical_severity_pass"] = 2
+        stats["total_critical_severity_fail"] = 4
+        stats["total_high_severity_fail"] = 1
+        stats["total_high_severity_pass"] = 1
+        stats["total_medium_severity_fail"] = 2
+        stats["total_medium_severity_pass"] = 1
+        stats["total_low_severity_fail"] = 2
+        stats["total_low_severity_pass"] = 3
        stats["resources_count"] = 20
        stats["findings_count"] = 22

@@ -284,6 +368,19 @@ class TestSlackIntegration:
                    "text": f"\n:white_check_mark: *{stats['total_pass']} Passed findings* ({round(stats['total_pass'] / stats['findings_count'] * 100 , 2)}%)\n",
                },
            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": (
+                        "*Severities:*\n"
+                        "• *Critical:* 2 "
+                        "• *High:* 1 "
+                        "• *Medium:* 1 "
+                        "• *Low:* 3"
+                    ),
+                },
+            },
            {
                "type": "section",
                "text": {
@@ -291,6 +388,19 @@ class TestSlackIntegration:
                    "text": f"\n:x: *{stats['total_fail']} Failed findings* ({round(stats['total_fail'] / stats['findings_count'] * 100 , 2)}%)\n ",
                },
            },
+            {
+                "type": "section",
+                "text": {
+                    "type": "mrkdwn",
+                    "text": (
+                        "*Severities:*\n"
+                        "• *Critical:* 4 "
+                        "• *High:* 1 "
+                        "• *Medium:* 2 "
+                        "• *Low:* 2"
+                    ),
+                },
+            },
            {
                "type": "section",
                "text": {