From 1f4316e9ddabac93812abdad6e8d7f3e6cdf3418 Mon Sep 17 00:00:00 2001 From: Pepe Fagoaga Date: Thu, 14 Mar 2024 15:04:47 +0100 Subject: [PATCH] chore(ocsf): add OCSF 1.1 and organize code (#3517) Co-authored-by: Sergio Garcia --- poetry.lock | 106 ++-- prowler/__main__.py | 9 + prowler/lib/outputs/common.py | 142 ++++++ prowler/lib/outputs/common_models.py | 77 +++ .../aws_well_architected_framework.py | 2 +- prowler/lib/outputs/compliance/cis_aws.py | 2 +- prowler/lib/outputs/compliance/cis_gcp.py | 2 +- prowler/lib/outputs/compliance/compliance.py | 33 ++ .../lib/outputs/compliance/ens_rd2022_aws.py | 2 +- prowler/lib/outputs/compliance/generic.py | 2 +- .../outputs/compliance/iso27001_2013_aws.py | 2 +- .../outputs/compliance/mitre_attack_aws.py | 3 +- prowler/lib/outputs/compliance/models.py | 181 +++++++ prowler/lib/outputs/csv/csv.py | 141 ------ prowler/lib/outputs/file_descriptors.py | 14 +- prowler/lib/outputs/json.py | 363 -------------- prowler/lib/outputs/json/__init__.py | 0 prowler/lib/outputs/json/json.py | 39 ++ prowler/lib/outputs/json_asff/__init__.py | 0 prowler/lib/outputs/json_asff/json_asff.py | 121 +++++ prowler/lib/outputs/json_asff/models.py | 51 ++ prowler/lib/outputs/json_ocsf/__init__.py | 0 prowler/lib/outputs/json_ocsf/json_ocsf.py | 143 ++++++ prowler/lib/outputs/models.py | 469 ------------------ prowler/lib/outputs/outputs.py | 75 ++- prowler/lib/outputs/utils.py | 86 ++++ prowler/providers/aws/aws_provider.py | 10 +- prowler/providers/aws/lib/arn/error.py | 2 +- .../providers/aws/lib/mutelist/mutelist.py | 2 +- .../aws/lib/organizations/organizations.py | 20 +- .../lib/quick_inventory/quick_inventory.py | 2 +- .../aws/lib/security_hub/security_hub.py | 4 +- prowler/providers/aws/models.py | 11 +- prowler/providers/azure/azure_provider.py | 2 +- pyproject.toml | 1 + tests/lib/cli/parser_test.py | 54 +- tests/lib/outputs/outputs_test.py | 22 - 37 files changed, 1074 insertions(+), 1121 deletions(-) create mode 100644 prowler/lib/outputs/common.py create mode 100644 prowler/lib/outputs/common_models.py create mode 100644 prowler/lib/outputs/compliance/models.py delete mode 100644 prowler/lib/outputs/json.py create mode 100644 prowler/lib/outputs/json/__init__.py create mode 100644 prowler/lib/outputs/json/json.py create mode 100644 prowler/lib/outputs/json_asff/__init__.py create mode 100644 prowler/lib/outputs/json_asff/json_asff.py create mode 100644 prowler/lib/outputs/json_asff/models.py create mode 100644 prowler/lib/outputs/json_ocsf/__init__.py create mode 100644 prowler/lib/outputs/json_ocsf/json_ocsf.py delete mode 100644 prowler/lib/outputs/models.py create mode 100644 prowler/lib/outputs/utils.py diff --git a/poetry.lock b/poetry.lock index 3d6357be74..67409dc6c8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -235,13 +235,13 @@ cryptography = "*" [[package]] name = "aws-sam-translator" -version = "1.85.0" +version = "1.86.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" optional = false python-versions = ">=3.8, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.85.0.tar.gz", hash = "sha256:e41938affa128fb5bde5e1989b260bf539a96369bba3faf316ce66651351df39"}, - {file = "aws_sam_translator-1.85.0-py3-none-any.whl", hash = "sha256:e8c69a4db7279421ff6c3579cd4d43395fe9b6781f50416528e984be68e25481"}, + {file = "aws-sam-translator-1.86.0.tar.gz", hash = "sha256:a748dcd7886024cb7586abbbdbabe8c787c44c6547bb6602879d7bb8a6934d05"}, + {file = "aws_sam_translator-1.86.0-py3-none-any.whl", hash = "sha256:97a44e5ac8b0d141c31f4ed35c57aa94429a0e6cef7fe989831c9a1c40455473"}, ] [package.dependencies] @@ -255,13 +255,13 @@ dev = ["black (==23.10.1)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverle [[package]] name = "aws-xray-sdk" -version = "2.12.1" +version = "2.13.0" description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." optional = false python-versions = ">=3.7" files = [ - {file = "aws-xray-sdk-2.12.1.tar.gz", hash = "sha256:0bbfdbc773cfef4061062ac940b85e408297a2242f120bcdfee2593209b1e432"}, - {file = "aws_xray_sdk-2.12.1-py2.py3-none-any.whl", hash = "sha256:f6803832dc08d18cc265e2327a69bfa9ee41c121fac195edc9745d04b7a566c3"}, + {file = "aws-xray-sdk-2.13.0.tar.gz", hash = "sha256:816186126917bc35ae4e6e2f304702a43d494ecef34a39e6330f5018bdecc9f5"}, + {file = "aws_xray_sdk-2.13.0-py2.py3-none-any.whl", hash = "sha256:d18604a8688b4bed03ce4a858cc9acd72b71400e085bf7512fc31cf657ca85f9"}, ] [package.dependencies] @@ -1171,6 +1171,26 @@ files = [ graph = ["objgraph (>=1.7.2)"] profile = ["gprof2dot (>=2022.7.29)"] +[[package]] +name = "dnspython" +version = "2.6.1" +description = "DNS toolkit" +optional = false +python-versions = ">=3.8" +files = [ + {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, + {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, +] + +[package.extras] +dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "sphinx (>=7.2.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] +dnssec = ["cryptography (>=41)"] +doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] +doq = ["aioquic (>=0.9.25)"] +idna = ["idna (>=3.6)"] +trio = ["trio (>=0.23)"] +wmi = ["wmi (>=1.5.1)"] + [[package]] name = "docker" version = "7.0.0" @@ -1213,6 +1233,21 @@ conda = ["pyyaml"] pipenv = ["pipenv"] poetry = ["poetry"] +[[package]] +name = "email-validator" +version = "2.1.0.post1" +description = "A robust email address syntax and deliverability validation library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "email_validator-2.1.0.post1-py3-none-any.whl", hash = "sha256:c973053efbeddfef924dc0bd93f6e77a1ea7ee0fce935aea7103c7a3d6d2d637"}, + {file = "email_validator-2.1.0.post1.tar.gz", hash = "sha256:a4b0bd1cf55f073b924258d19321b1f3aa74b4b5a71a42c305575dba920e1a44"}, +] + +[package.dependencies] +dnspython = ">=2.0.0" +idna = ">=2.0.0" + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -1463,13 +1498,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.28.1" +version = "2.28.2" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.28.1.tar.gz", hash = "sha256:34fc3046c257cedcf1622fc4b31fc2be7923d9b4d44973d481125ecc50d83885"}, - {file = "google_auth-2.28.1-py2.py3-none-any.whl", hash = "sha256:25141e2d7a14bfcba945f5e9827f98092716e99482562f15306e5b026e21aa72"}, + {file = "google-auth-2.28.2.tar.gz", hash = "sha256:80b8b4969aa9ed5938c7828308f20f035bc79f9d8fb8120bf9dc8db20b41ba30"}, + {file = "google_auth-2.28.2-py2.py3-none-any.whl", hash = "sha256:9fd67bbcd40f16d9d42f950228e9cf02a2ded4ae49198b27432d0cded5a74c38"}, ] [package.dependencies] @@ -1501,13 +1536,13 @@ httplib2 = ">=0.19.0" [[package]] name = "googleapis-common-protos" -version = "1.62.0" +version = "1.63.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, - {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, + {file = "googleapis-common-protos-1.63.0.tar.gz", hash = "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e"}, + {file = "googleapis_common_protos-1.63.0-py2.py3-none-any.whl", hash = "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"}, ] [package.dependencies] @@ -2737,13 +2772,13 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -2957,6 +2992,21 @@ files = [ {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, ] +[[package]] +name = "py-ocsf-models" +version = "0.0.3" +description = "This is a Python implementation of the OCSF models. The models are used to represent the data of the OCSF Schema defined in https://schema.ocsf.io/." +optional = false +python-versions = ">=3.9,<3.13" +files = [ + {file = "py_ocsf_models-0.0.3-py3-none-any.whl", hash = "sha256:f4a47a1dbd502098a6016903cae328c29a52b2348a6d43e699889df98333f8ba"}, + {file = "py_ocsf_models-0.0.3.tar.gz", hash = "sha256:46de6cd03e2b743984c8e394544b7184f172fab7aea62f5133e13acf28a2036e"}, +] + +[package.dependencies] +email-validator = "2.1.0.post1" +pydantic = "1.10.14" + [[package]] name = "py-partiql-parser" version = "0.5.1" @@ -3536,13 +3586,13 @@ requests = ">=1.0.0" [[package]] name = "requests-oauthlib" -version = "1.3.1" +version = "1.4.0" description = "OAuthlib authentication support for Requests." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, - {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, + {file = "requests-oauthlib-1.4.0.tar.gz", hash = "sha256:acee623221e4a39abcbb919312c8ff04bd44e7e417087fb4bd5e2a2f53d5e79a"}, + {file = "requests_oauthlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:7a3130d94a17520169e38db6c8d75f2c974643788465ecc2e4b36d288bf13033"}, ] [package.dependencies] @@ -3901,18 +3951,18 @@ contextlib2 = ">=0.5.5" [[package]] name = "setuptools" -version = "69.1.1" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, - {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -4439,20 +4489,20 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.17.0" +version = "3.18.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, - {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, + {file = "zipp-3.18.0-py3-none-any.whl", hash = "sha256:c1bb803ed69d2cce2373152797064f7e79bc43f0a3748eb494096a867e0ebf79"}, + {file = "zipp-3.18.0.tar.gz", hash = "sha256:df8d042b02765029a09b157efd8e820451045890acc30f8e37dd2f94a060221f"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "7afdc0cc5591b2c397142e9ba88d797d81980a8d3b05b24c8c0bcc60e7e20d9e" +content-hash = "2bbb6bfb3c931d8d98d4cb5c1d5642073de26932c0bc73a188fe77c922707d0e" diff --git a/prowler/__main__.py b/prowler/__main__.py index 8ebcc5c528..fc5f140f80 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -33,6 +33,7 @@ from prowler.lib.check.custom_checks_metadata import ( from prowler.lib.cli.parser import ProwlerArgumentParser from prowler.lib.logger import logger, set_logging_config from prowler.lib.outputs.compliance.compliance import display_compliance_table +from prowler.lib.outputs.json.json import close_json from prowler.lib.outputs.outputs import extract_findings_statistics from prowler.lib.outputs.summary_table import display_summary_table from prowler.providers.aws.lib.s3.s3 import send_to_s3_bucket @@ -221,6 +222,14 @@ def prowler(): if args.output_modes: for mode in args.output_modes: + # Close json file if exists + if "json" in mode: + close_json( + global_provider.output_options.output_filename, + args.output_directory, + mode, + ) + # Send output to S3 if needed (-B / -D) if provider == "aws" and ( args.output_bucket or args.output_bucket_no_assume diff --git a/prowler/lib/outputs/common.py b/prowler/lib/outputs/common.py new file mode 100644 index 0000000000..d6c3b1512b --- /dev/null +++ b/prowler/lib/outputs/common.py @@ -0,0 +1,142 @@ +from operator import attrgetter + +from prowler.config.config import timestamp +from prowler.lib.logger import logger +from prowler.lib.outputs.common_models import FindingOutput +from prowler.lib.outputs.utils import unroll_list, unroll_tags +from prowler.lib.utils.utils import outputs_unix_timestamp + + +def get_provider_data_mapping(provider) -> dict: + data = {} + for generic_field, provider_field in provider.get_output_mapping.items(): + try: + provider_value = attrgetter(provider_field)(provider) + data[generic_field] = provider_value + except AttributeError as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + data[generic_field] = None + + return data + + +def generate_provider_output(provider, finding, csv_data) -> FindingOutput: + """ + generate_provider_output returns the provider's Finding output model + """ + # TODO: we have to standardize this between the above mapping and the provider.get_output_mapping() + try: + if provider.type == "aws": + # TODO: probably Organization UID is without the account id + csv_data["auth_method"] = f"profile: {csv_data['auth_method']}" + csv_data["resource_name"] = finding.resource_id + csv_data["resource_uid"] = finding.resource_arn + csv_data["region"] = finding.region + + elif provider.type == "azure": + # TODO: we should show the authentication method used I think + csv_data["auth_method"] = ( + f"{provider.identity.identity_type}: {provider.identity.identity_id}" + ) + + csv_data["account_uid"] = provider.identity.subscriptions[ + finding.subscription + ] + csv_data["account_name"] = finding.subscription + # Get the first tenant domain ID, just in case + csv_data["account_organization_uid"] = csv_data["account_organization_uid"][ + 0 + ] + csv_data["resource_name"] = finding.resource_name + csv_data["resource_uid"] = finding.resource_id + # TODO: pending to get location from Azure resources (finding.location) + csv_data["region"] = "" + + elif provider.type == "gcp": + csv_data["auth_method"] = f"Principal: {csv_data['auth_method']}" + csv_data["account_uid"] = provider.projects[finding.project_id].number + csv_data["account_name"] = provider.projects[finding.project_id].name + csv_data["account_tags"] = provider.projects[finding.project_id].labels + csv_data["resource_name"] = finding.resource_name + csv_data["resource_uid"] = finding.resource_id + csv_data["region"] = finding.location + + if ( + provider.projects + and finding.project_id in provider.projects + and getattr(provider.projects[finding.project_id], "organization") + ): + csv_data["account_organization_uid"] = provider.projects[ + finding.project_id + ].organization.id + # TODO: for now is None since we don't retrieve that data + csv_data["account_organization"] = provider.projects[ + finding.project_id + ].organization.display_name + + elif provider.type == "kubernetes": + if provider.identity.context == "In-Cluster": + csv_data["auth_method"] = "in-cluster" + else: + csv_data["auth_method"] = "kubeconfig" + csv_data["resource_name"] = finding.resource_name + csv_data["resource_uid"] = finding.resource_id + csv_data["account_name"] = f"context: {provider.identity.context}" + csv_data["region"] = f"namespace: {finding.namespace}" + + # Finding Unique ID + # TODO: move this to a function + # TODO: in Azure, GCP and K8s there are fidings without resource_name + csv_data["finding_uid"] = ( + f"prowler-{provider.type}-{finding.check_metadata.CheckID}-{csv_data['account_uid']}-{csv_data['region']}-{csv_data['resource_name']}" + ) + + finding_output = FindingOutput(**csv_data) + + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + else: + return finding_output + + +def fill_common_finding_data(finding: dict, unix_timestamp: bool) -> dict: + finding_data = { + "timestamp": outputs_unix_timestamp(unix_timestamp, timestamp), + "check_id": finding.check_metadata.CheckID, + "check_title": finding.check_metadata.CheckTitle, + "check_type": ",".join(finding.check_metadata.CheckType), + "status": finding.status, + "status_extended": finding.status_extended, + "service_name": finding.check_metadata.ServiceName, + "subservice_name": finding.check_metadata.SubServiceName, + "severity": finding.check_metadata.Severity, + "resource_type": finding.check_metadata.ResourceType, + "resource_details": finding.resource_details, + "resource_tags": unroll_tags(finding.resource_tags), + "description": finding.check_metadata.Description, + "risk": finding.check_metadata.Risk, + "related_url": finding.check_metadata.RelatedUrl, + "remediation_recommendation_text": ( + finding.check_metadata.Remediation.Recommendation.Text + ), + "remediation_recommendation_url": ( + finding.check_metadata.Remediation.Recommendation.Url + ), + "remediation_code_nativeiac": ( + finding.check_metadata.Remediation.Code.NativeIaC + ), + "remediation_code_terraform": ( + finding.check_metadata.Remediation.Code.Terraform + ), + "remediation_code_cli": (finding.check_metadata.Remediation.Code.CLI), + "remediation_code_other": (finding.check_metadata.Remediation.Code.Other), + "categories": unroll_list(finding.check_metadata.Categories), + "depends_on": unroll_list(finding.check_metadata.DependsOn), + "related_to": unroll_list(finding.check_metadata.RelatedTo), + "notes": finding.check_metadata.Notes, + } + return finding_data diff --git a/prowler/lib/outputs/common_models.py b/prowler/lib/outputs/common_models.py new file mode 100644 index 0000000000..b040adb122 --- /dev/null +++ b/prowler/lib/outputs/common_models.py @@ -0,0 +1,77 @@ +from datetime import datetime +from enum import Enum +from typing import Optional + +from pydantic import BaseModel + +from prowler.config.config import prowler_version + + +class Status(str, Enum): + PASS = "PASS" + FAIL = "FAIL" + MANUAL = "MANUAL" + + +class Severity(str, Enum): + critical = "critical" + high = "high" + medium = "medium" + low = "low" + informational = "informational" + + +class FindingOutput(BaseModel): + """ + FindingOutput generates a finding's output. It can be written to CSV or another format doing the mapping. + + This is the base finding output model for every provider. + """ + + auth_method: str + timestamp: datetime + account_uid: str + # Optional since depends on permissions + account_name: Optional[str] + # Optional since depends on permissions + account_email: Optional[str] + # Optional since depends on permissions + account_organization_uid: Optional[str] + # Optional since depends on permissions + account_organization_name: Optional[str] + # Optional since depends on permissions + account_tags: Optional[str] + finding_uid: str + provider: str + check_id: str + check_title: str + check_type: str + status: Status + status_extended: str + muted: bool = False + service_name: str + subservice_name: str + severity: Severity + resource_type: str + resource_uid: str + resource_name: str + resource_details: str + resource_tags: str + # Only present for AWS and Azure + partition: Optional[str] + region: str + description: str + risk: str + related_url: str + remediation_recommendation_text: str + remediation_recommendation_url: str + remediation_code_nativeiac: str + remediation_code_terraform: str + remediation_code_cli: str + remediation_code_other: str + compliance: str + categories: str + depends_on: str + related_to: str + notes: str + prowler_version: str = prowler_version diff --git a/prowler/lib/outputs/compliance/aws_well_architected_framework.py b/prowler/lib/outputs/compliance/aws_well_architected_framework.py index eecf9da798..ce8d8bcbc5 100644 --- a/prowler/lib/outputs/compliance/aws_well_architected_framework.py +++ b/prowler/lib/outputs/compliance/aws_well_architected_framework.py @@ -2,8 +2,8 @@ from csv import DictWriter from prowler.config.config import timestamp from prowler.lib.logger import logger +from prowler.lib.outputs.compliance.models import Check_Output_CSV_AWS_Well_Architected from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_CSV_AWS_Well_Architected from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/cis_aws.py b/prowler/lib/outputs/compliance/cis_aws.py index f6409b3b4a..ae90227ccd 100644 --- a/prowler/lib/outputs/compliance/cis_aws.py +++ b/prowler/lib/outputs/compliance/cis_aws.py @@ -1,6 +1,6 @@ from prowler.config.config import timestamp +from prowler.lib.outputs.compliance.models import Check_Output_CSV_AWS_CIS from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_CSV_AWS_CIS from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/cis_gcp.py b/prowler/lib/outputs/compliance/cis_gcp.py index 8749915529..eb8b09162a 100644 --- a/prowler/lib/outputs/compliance/cis_gcp.py +++ b/prowler/lib/outputs/compliance/cis_gcp.py @@ -1,6 +1,6 @@ from prowler.config.config import timestamp +from prowler.lib.outputs.compliance.models import Check_Output_CSV_GCP_CIS from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_CSV_GCP_CIS from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/compliance.py b/prowler/lib/outputs/compliance/compliance.py index 03267716b4..80fdbad7d0 100644 --- a/prowler/lib/outputs/compliance/compliance.py +++ b/prowler/lib/outputs/compliance/compliance.py @@ -463,3 +463,36 @@ def display_compliance_table( f"{error.__class__.__name__}:{error.__traceback__.tb_lineno} -- {error}" ) sys.exit(1) + + +def get_check_compliance(finding, provider_type, output_options) -> dict: + """get_check_compliance returns a map with the compliance framework as key and the requirements where the finding's check is present. + + Example: + + { + "CIS-1.4": ["2.1.3"], + "CIS-1.5": ["2.1.3"], + } + """ + try: + check_compliance = {} + # We have to retrieve all the check's compliance requirements + if finding.check_metadata.CheckID in output_options.bulk_checks_metadata: + for compliance in output_options.bulk_checks_metadata[ + finding.check_metadata.CheckID + ].Compliance: + compliance_fw = compliance.Framework + if compliance.Version: + compliance_fw = f"{compliance_fw}-{compliance.Version}" + if compliance.Provider == provider_type.upper(): + if compliance_fw not in check_compliance: + check_compliance[compliance_fw] = [] + for requirement in compliance.Requirements: + check_compliance[compliance_fw].append(requirement.Id) + return check_compliance + except Exception as error: + logger.critical( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}] -- {error}" + ) + sys.exit(1) diff --git a/prowler/lib/outputs/compliance/ens_rd2022_aws.py b/prowler/lib/outputs/compliance/ens_rd2022_aws.py index 281742201d..149300f0b9 100644 --- a/prowler/lib/outputs/compliance/ens_rd2022_aws.py +++ b/prowler/lib/outputs/compliance/ens_rd2022_aws.py @@ -1,8 +1,8 @@ from csv import DictWriter from prowler.config.config import timestamp +from prowler.lib.outputs.compliance.models import Check_Output_CSV_ENS_RD2022 from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_CSV_ENS_RD2022 from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/generic.py b/prowler/lib/outputs/compliance/generic.py index d3e85af922..83d5abada5 100644 --- a/prowler/lib/outputs/compliance/generic.py +++ b/prowler/lib/outputs/compliance/generic.py @@ -1,8 +1,8 @@ from csv import DictWriter from prowler.config.config import timestamp +from prowler.lib.outputs.compliance.models import Check_Output_CSV_Generic_Compliance from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_CSV_Generic_Compliance from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/iso27001_2013_aws.py b/prowler/lib/outputs/compliance/iso27001_2013_aws.py index f796ea18df..60e5799a68 100644 --- a/prowler/lib/outputs/compliance/iso27001_2013_aws.py +++ b/prowler/lib/outputs/compliance/iso27001_2013_aws.py @@ -1,8 +1,8 @@ from csv import DictWriter from prowler.config.config import timestamp +from prowler.lib.outputs.compliance.models import Check_Output_CSV_AWS_ISO27001_2013 from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_CSV_AWS_ISO27001_2013 from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/mitre_attack_aws.py b/prowler/lib/outputs/compliance/mitre_attack_aws.py index 34ca92e3be..6adacbb1a6 100644 --- a/prowler/lib/outputs/compliance/mitre_attack_aws.py +++ b/prowler/lib/outputs/compliance/mitre_attack_aws.py @@ -1,8 +1,9 @@ from csv import DictWriter from prowler.config.config import timestamp +from prowler.lib.outputs.compliance.models import Check_Output_MITRE_ATTACK from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.models import Check_Output_MITRE_ATTACK, unroll_list +from prowler.lib.outputs.utils import unroll_list from prowler.lib.utils.utils import outputs_unix_timestamp diff --git a/prowler/lib/outputs/compliance/models.py b/prowler/lib/outputs/compliance/models.py new file mode 100644 index 0000000000..97f22e287b --- /dev/null +++ b/prowler/lib/outputs/compliance/models.py @@ -0,0 +1,181 @@ +from typing import Optional + +from pydantic import BaseModel + + +# TODO: move this to outputs//models.py +class Check_Output_MITRE_ATTACK(BaseModel): + """ + Check_Output_MITRE_ATTACK generates a finding's output in CSV MITRE ATTACK format. + """ + + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Id: str + Requirements_Name: str + Requirements_Description: str + Requirements_Tactics: str + Requirements_SubTechniques: str + Requirements_Platforms: str + Requirements_TechniqueURL: str + Requirements_Attributes_AWSServices: str + Requirements_Attributes_Categories: str + Requirements_Attributes_Values: str + Requirements_Attributes_Comments: str + Status: str + StatusExtended: str + ResourceId: str + CheckId: str + + +class Check_Output_CSV_ENS_RD2022(BaseModel): + """ + Check_Output_CSV_ENS_RD2022 generates a finding's output in CSV ENS RD2022 format. + """ + + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Id: str + Requirements_Description: str + Requirements_Attributes_IdGrupoControl: str + Requirements_Attributes_Marco: str + Requirements_Attributes_Categoria: str + Requirements_Attributes_DescripcionControl: str + Requirements_Attributes_Nivel: str + Requirements_Attributes_Tipo: str + Requirements_Attributes_Dimensiones: str + Status: str + StatusExtended: str + ResourceId: str + CheckId: str + + +class Check_Output_CSV_AWS_CIS(BaseModel): + """ + Check_Output_CSV_CIS generates a finding's output in CSV CIS format. + """ + + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Id: str + Requirements_Description: str + Requirements_Attributes_Section: str + Requirements_Attributes_Profile: str + Requirements_Attributes_AssessmentStatus: str + Requirements_Attributes_Description: str + Requirements_Attributes_RationaleStatement: str + Requirements_Attributes_ImpactStatement: str + Requirements_Attributes_RemediationProcedure: str + Requirements_Attributes_AuditProcedure: str + Requirements_Attributes_AdditionalInformation: str + Requirements_Attributes_References: str + Status: str + StatusExtended: str + ResourceId: str + CheckId: str + + +class Check_Output_CSV_GCP_CIS(BaseModel): + """ + Check_Output_CSV_CIS generates a finding's output in CSV CIS format. + """ + + Provider: str + Description: str + ProjectId: str + Location: str + AssessmentDate: str + Requirements_Id: str + Requirements_Description: str + Requirements_Attributes_Section: str + Requirements_Attributes_Profile: str + Requirements_Attributes_AssessmentStatus: str + Requirements_Attributes_Description: str + Requirements_Attributes_RationaleStatement: str + Requirements_Attributes_ImpactStatement: str + Requirements_Attributes_RemediationProcedure: str + Requirements_Attributes_AuditProcedure: str + Requirements_Attributes_AdditionalInformation: str + Requirements_Attributes_References: str + Status: str + StatusExtended: str + ResourceId: str + ResourceName: str + CheckId: str + + +class Check_Output_CSV_Generic_Compliance(BaseModel): + """ + Check_Output_CSV_Generic_Compliance generates a finding's output in CSV Generic Compliance format. + """ + + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Id: str + Requirements_Description: str + Requirements_Attributes_Section: Optional[str] + Requirements_Attributes_SubSection: Optional[str] + Requirements_Attributes_SubGroup: Optional[str] + Requirements_Attributes_Service: Optional[str] + Requirements_Attributes_Type: Optional[str] + Status: str + StatusExtended: str + ResourceId: str + CheckId: str + + +class Check_Output_CSV_AWS_Well_Architected(BaseModel): + """ + Check_Output_CSV_AWS_Well_Architected generates a finding's output in CSV AWS Well Architected Compliance format. + """ + + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Attributes_Name: str + Requirements_Attributes_WellArchitectedQuestionId: str + Requirements_Attributes_WellArchitectedPracticeId: str + Requirements_Attributes_Section: str + Requirements_Attributes_SubSection: Optional[str] + Requirements_Attributes_LevelOfRisk: str + Requirements_Attributes_AssessmentMethod: str + Requirements_Attributes_Description: str + Requirements_Attributes_ImplementationGuidanceUrl: str + Status: str + StatusExtended: str + ResourceId: str + CheckId: str + + +class Check_Output_CSV_AWS_ISO27001_2013(BaseModel): + """ + Check_Output_CSV_AWS_ISO27001_2013 generates a finding's output in CSV AWS ISO27001 Compliance format. + """ + + Provider: str + Description: str + AccountId: str + Region: str + AssessmentDate: str + Requirements_Attributes_Category: str + Requirements_Attributes_Objetive_ID: str + Requirements_Attributes_Objetive_Name: str + Requirements_Attributes_Check_Summary: str + Status: str + StatusExtended: str + ResourceId: str + CheckId: str diff --git a/prowler/lib/outputs/csv/csv.py b/prowler/lib/outputs/csv/csv.py index 4371a593db..02dc9cf503 100644 --- a/prowler/lib/outputs/csv/csv.py +++ b/prowler/lib/outputs/csv/csv.py @@ -1,147 +1,6 @@ from csv import DictWriter -from operator import attrgetter from typing import Any -from prowler.config.config import timestamp -from prowler.lib.logger import logger -from prowler.lib.outputs.csv.models import CSVRow -from prowler.lib.outputs.models import unroll_list, unroll_tags -from prowler.lib.utils.utils import outputs_unix_timestamp - - -def get_provider_data_mapping(provider) -> dict: - data = {} - for generic_field, provider_field in provider.get_output_mapping.items(): - try: - provider_value = attrgetter(provider_field)(provider) - data[generic_field] = provider_value - except AttributeError as error: - logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - data[generic_field] = None - - return data - - -def fill_common_data_csv(finding: dict, unix_timestamp: bool) -> dict: - data = { - "timestamp": outputs_unix_timestamp(unix_timestamp, timestamp), - "check_id": finding.check_metadata.CheckID, - "check_title": finding.check_metadata.CheckTitle, - "check_type": ",".join(finding.check_metadata.CheckType), - "status": finding.status, - "status_extended": finding.status_extended, - "service_name": finding.check_metadata.ServiceName, - "subservice_name": finding.check_metadata.SubServiceName, - "severity": finding.check_metadata.Severity, - "resource_type": finding.check_metadata.ResourceType, - "resource_details": finding.resource_details, - "resource_tags": unroll_tags(finding.resource_tags), - "description": finding.check_metadata.Description, - "risk": finding.check_metadata.Risk, - "related_url": finding.check_metadata.RelatedUrl, - "remediation_recommendation_text": ( - finding.check_metadata.Remediation.Recommendation.Text - ), - "remediation_recommendation_url": ( - finding.check_metadata.Remediation.Recommendation.Url - ), - "remediation_code_nativeiac": ( - finding.check_metadata.Remediation.Code.NativeIaC - ), - "remediation_code_terraform": ( - finding.check_metadata.Remediation.Code.Terraform - ), - "remediation_code_cli": (finding.check_metadata.Remediation.Code.CLI), - "remediation_code_other": (finding.check_metadata.Remediation.Code.Other), - "categories": unroll_list(finding.check_metadata.Categories), - "depends_on": unroll_list(finding.check_metadata.DependsOn), - "related_to": unroll_list(finding.check_metadata.RelatedTo), - "notes": finding.check_metadata.Notes, - } - return data - - -def generate_provider_output_csv(provider, finding, csv_data): - """ - generate_provider_output_csv creates the provider's CSV output - """ - # TODO: we have to standardize this between the above mapping and the provider.get_output_mapping() - try: - if provider.type == "aws": - csv_data["auth_method"] = f"profile: {csv_data['auth_method']}" - csv_data["resource_name"] = finding.resource_id - csv_data["resource_uid"] = finding.resource_arn - csv_data["region"] = finding.region - - elif provider.type == "azure": - # TODO: we should show the authentication method used I think - csv_data["auth_method"] = ( - f"{provider.identity.identity_type}: {provider.identity.identity_id}" - ) - - csv_data["account_uid"] = provider.identity.subscriptions[ - finding.subscription - ] - csv_data["account_name"] = finding.subscription - # Get the first tenant domain ID, just in case - csv_data["account_organization_uid"] = csv_data["account_organization_uid"][ - 0 - ] - csv_data["resource_name"] = finding.resource_name - csv_data["resource_uid"] = finding.resource_id - # TODO: pending to get location from Azure resources (finding.location) - csv_data["region"] = "" - - elif provider.type == "gcp": - csv_data["auth_method"] = f"Account: {csv_data['auth_method']}" - csv_data["account_uid"] = provider.projects[finding.project_id].number - csv_data["account_name"] = provider.projects[finding.project_id].name - csv_data["account_tags"] = provider.projects[finding.project_id].labels - csv_data["resource_name"] = finding.resource_name - csv_data["resource_uid"] = finding.resource_id - csv_data["region"] = finding.location - - if ( - provider.projects - and finding.project_id in provider.projects - and getattr(provider.projects[finding.project_id], "organization") - ): - csv_data["account_organization_uid"] = provider.projects[ - finding.project_id - ].organization.id - # TODO: for now is None since we don't retrieve that data - csv_data["account_organization"] = provider.projects[ - finding.project_id - ].organization.display_name - - elif provider.type == "kubernetes": - if provider.identity.context == "In-Cluster": - csv_data["auth_method"] = "in-cluster" - else: - csv_data["auth_method"] = "kubeconfig" - csv_data["resource_name"] = finding.resource_name - csv_data["resource_uid"] = finding.resource_id - csv_data["account_name"] = f"context: {provider.identity.context}" - csv_data["region"] = f"namespace: {finding.namespace}" - - # Finding Unique ID - # TODO: move this to a function - # TODO: in Azure, GCP and K8s there are fidings without resource_name - csv_data["finding_uid"] = ( - f"prowler-{provider.type}-{finding.check_metadata.CheckID}-{csv_data['account_uid']}-{csv_data['region']}-{csv_data['resource_name']}" - ) - - finding_output = CSVRow(**csv_data) - - except Exception as error: - logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - else: - return finding_output - def write_csv(file_descriptor, headers, row): csv_writer = DictWriter( diff --git a/prowler/lib/outputs/file_descriptors.py b/prowler/lib/outputs/file_descriptors.py index 0719a4c66c..1cb70a55b8 100644 --- a/prowler/lib/outputs/file_descriptors.py +++ b/prowler/lib/outputs/file_descriptors.py @@ -8,9 +8,8 @@ from prowler.config.config import ( json_ocsf_file_suffix, ) from prowler.lib.logger import logger -from prowler.lib.outputs.csv.csv import generate_csv_fields -from prowler.lib.outputs.csv.models import CSVRow -from prowler.lib.outputs.models import ( +from prowler.lib.outputs.common_models import FindingOutput +from prowler.lib.outputs.compliance.models import ( Check_Output_CSV_AWS_CIS, Check_Output_CSV_AWS_ISO27001_2013, Check_Output_CSV_AWS_Well_Architected, @@ -19,13 +18,14 @@ from prowler.lib.outputs.models import ( Check_Output_CSV_Generic_Compliance, Check_Output_MITRE_ATTACK, ) +from prowler.lib.outputs.csv.csv import generate_csv_fields from prowler.lib.utils.utils import file_exists, open_file def initialize_file_descriptor( - filename: str, output_mode: str, format: Any = CSVRow + filename: str, output_mode: str, format: Any = FindingOutput ) -> TextIOWrapper: - """Open/Create the output file. If needed include headers or the required format, by default will use the CSVRow""" + """Open/Create the output file. If needed include headers or the required format, by default will use the FindingOutput""" try: if file_exists(filename): file_descriptor = open_file( @@ -61,7 +61,7 @@ def fill_file_descriptors(output_modes, output_directory, output_filename, provi for output_mode in output_modes: if output_mode == "csv": filename = f"{output_directory}/{output_filename}{csv_file_suffix}" - output_model = CSVRow + output_model = FindingOutput file_descriptor = initialize_file_descriptor( filename, output_mode, @@ -87,7 +87,6 @@ def fill_file_descriptors(output_modes, output_directory, output_filename, provi file_descriptor = initialize_file_descriptor( filename, output_mode, - provider, Check_Output_CSV_Generic_Compliance, ) file_descriptors.update({output_mode: file_descriptor}) @@ -154,7 +153,6 @@ def fill_file_descriptors(output_modes, output_directory, output_filename, provi file_descriptor = initialize_file_descriptor( filename, output_mode, - provider, Check_Output_CSV_Generic_Compliance, ) file_descriptors.update({output_mode: file_descriptor}) diff --git a/prowler/lib/outputs/json.py b/prowler/lib/outputs/json.py deleted file mode 100644 index f72bd65bbd..0000000000 --- a/prowler/lib/outputs/json.py +++ /dev/null @@ -1,363 +0,0 @@ -import os -import sys - -from prowler.config.config import ( - json_asff_file_suffix, - json_file_suffix, - json_ocsf_file_suffix, - prowler_version, - timestamp, - timestamp_utc, -) -from prowler.lib.logger import logger -from prowler.lib.outputs.models import ( - Account, - Check_Output_JSON_OCSF, - Cloud, - Compliance, - Compliance_OCSF, - Feature, - Finding, - Group, - Metadata, - Organization, - Product, - ProductFields, - Remediation_OCSF, - Resource, - Resources, - Severity, - get_check_compliance, - unroll_dict_to_list, -) -from prowler.lib.utils.utils import hash_sha512, open_file, outputs_unix_timestamp - - -def fill_json_asff(finding_output, provider, finding, output_options): - try: - # Check if there are no resources in the finding - if finding.resource_arn == "": - if finding.resource_id == "": - finding.resource_id = "NONE_PROVIDED" - finding.resource_arn = finding.resource_id - # The following line cannot be changed because it is the format we use to generate unique findings for AWS Security Hub - # If changed some findings could be lost because the unique identifier will be different - finding_output.Id = f"prowler-{finding.check_metadata.CheckID}-{provider.identity.account}-{finding.region}-{hash_sha512(finding.resource_id)}" - finding_output.ProductArn = f"arn:{provider.identity.partition}:securityhub:{finding.region}::product/prowler/prowler" - finding_output.ProductFields = ProductFields( - ProviderVersion=prowler_version, ProwlerResourceName=finding.resource_arn - ) - finding_output.GeneratorId = "prowler-" + finding.check_metadata.CheckID - finding_output.AwsAccountId = provider.identity.account - finding_output.Types = finding.check_metadata.CheckType - finding_output.FirstObservedAt = finding_output.UpdatedAt = ( - finding_output.CreatedAt - ) = timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ") - finding_output.Severity = Severity( - Label=finding.check_metadata.Severity.upper() - ) - finding_output.Title = finding.check_metadata.CheckTitle - # Description should NOT be longer than 1024 characters - finding_output.Description = ( - (finding.status_extended[:1000] + "...") - if len(finding.status_extended) > 1000 - else finding.status_extended - ) - resource_tags = generate_json_asff_resource_tags(finding.resource_tags) - finding_output.Resources = [ - Resource( - Id=finding.resource_arn, - Type=finding.check_metadata.ResourceType, - Partition=provider.identity.partition, - Region=finding.region, - Tags=resource_tags, - ) - ] - # Iterate for each compliance framework - compliance_summary = [] - associated_standards = [] - check_compliance = get_check_compliance(finding, "aws", output_options) - for key, value in check_compliance.items(): - if ( - len(associated_standards) < 20 - ): # AssociatedStandards should NOT have more than 20 items - associated_standards.append({"StandardsId": key}) - item = f"{key} {' '.join(value)}" - if len(item) > 64: - item = item[0:63] - compliance_summary.append(item) - - # Ensures finding_status matches allowed values in ASFF - finding_status = generate_json_asff_status(finding.status) - - finding_output.Compliance = Compliance( - Status=finding_status, - AssociatedStandards=associated_standards, - RelatedRequirements=compliance_summary, - ) - # Fill Recommendation Url if it is blank - if not finding.check_metadata.Remediation.Recommendation.Url: - finding.check_metadata.Remediation.Recommendation.Url = "https://docs.aws.amazon.com/securityhub/latest/userguide/what-is-securityhub.html" - finding_output.Remediation = { - "Recommendation": finding.check_metadata.Remediation.Recommendation - } - - return finding_output - except Exception as error: - logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - - -def generate_json_asff_status(status: str) -> str: - json_asff_status = "" - if status == "PASS": - json_asff_status = "PASSED" - elif status == "FAIL": - json_asff_status = "FAILED" - elif status == "MUTED": - json_asff_status = "MUTED" - else: - json_asff_status = "NOT_AVAILABLE" - - return json_asff_status - - -def generate_json_asff_resource_tags(tags): - try: - resource_tags = {} - if tags and tags != [None]: - for tag in tags: - if "Key" in tag and "Value" in tag: - resource_tags[tag["Key"]] = tag["Value"] - else: - resource_tags.update(tag) - if len(resource_tags) == 0: - return None - else: - return None - return resource_tags - except Exception as error: - logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - - -def fill_json_ocsf(provider, finding, output_options) -> Check_Output_JSON_OCSF: - try: - resource_region = "" - resource_name = "" - resource_uid = "" - finding_uid = "" - project_uid = "" - resource_labels = finding.resource_tags if finding.resource_tags else [] - aws_account_name = "" - aws_org_uid = "" - account = None - org = None - profile = "" - if provider.type == "aws": - profile = ( - provider.identity.profile - if provider.identity.profile is not None - else "default" - ) - if ( - hasattr(provider, "organizations_metadata") - and provider.organizations_metadata - ): - aws_account_name = provider.organizations_metadata.account_details_name - aws_org_uid = provider.organizations_metadata.account_details_org - if finding.check_metadata.Provider == "aws": - account = Account( - name=aws_account_name, - uid=provider.identity.account, - ) - org = Organization( - name=aws_org_uid, - uid=aws_org_uid, - ) - resource_region = finding.region - resource_name = finding.resource_id - resource_uid = finding.resource_arn - finding_uid = f"prowler-{finding.check_metadata.Provider}-{finding.check_metadata.CheckID}-{provider.identity.account}-{finding.region}-{finding.resource_id}" - elif finding.check_metadata.Provider == "azure": - account = Account( - name=finding.subscription, - uid=finding.subscription, - ) - org = Organization( - name=provider.identity.tenant_domain, - uid=provider.identity.tenant_domain, - ) - resource_name = finding.resource_name - resource_uid = finding.resource_id - finding_uid = f"prowler-{finding.check_metadata.Provider}-{finding.check_metadata.CheckID}-{finding.subscription}-{finding.resource_id}" - elif finding.check_metadata.Provider == "gcp": - project_uid = finding.project_id - resource_region = finding.location.lower() - resource_name = finding.resource_name - resource_uid = finding.resource_id - finding_uid = f"prowler-{finding.check_metadata.Provider}-{finding.check_metadata.CheckID}-{finding.project_id}-{finding.resource_id}" - elif finding.check_metadata.Provider == "kubernetes": - resource_name = finding.resource_name - resource_uid = finding.resource_id - finding_uid = f"prowler-{finding.check_metadata.Provider}-{finding.check_metadata.CheckID}-{finding.namespace}-{finding.resource_id}" - cloud = Cloud( - provider=finding.check_metadata.Provider, - org=org, - account=account, - region=resource_region, - project_uid=project_uid, - ) - finding_ocsf = Finding( - title=finding.check_metadata.CheckTitle, - uid=finding_uid, - desc=finding.check_metadata.Description, - supporting_data={ - "Risk": finding.check_metadata.Risk, - "Notes": finding.check_metadata.Notes, - }, - related_events=finding.check_metadata.DependsOn - + finding.check_metadata.RelatedTo, - remediation=Remediation_OCSF( - kb_articles=list( - filter( - None, - [ - finding.check_metadata.Remediation.Code.NativeIaC, - finding.check_metadata.Remediation.Code.Terraform, - finding.check_metadata.Remediation.Code.CLI, - finding.check_metadata.Remediation.Code.Other, - finding.check_metadata.Remediation.Recommendation.Url, - ], - ) - ), - desc=finding.check_metadata.Remediation.Recommendation.Text, - ), - types=finding.check_metadata.CheckType, - src_url=finding.check_metadata.RelatedUrl, - ) - resources = [] - resources.append( - Resources( - group=Group(name=finding.check_metadata.ServiceName), - region=resource_region, - name=resource_name, - labels=resource_labels, - uid=resource_uid, - type=finding.check_metadata.ResourceType, - details=finding.resource_details, - ) - ) - metadata = Metadata( - product=Product( - feature=Feature( - uid=finding.check_metadata.CheckID, - name=finding.check_metadata.CheckID, - ) - ), - original_time=outputs_unix_timestamp( - output_options.unix_timestamp, timestamp - ), - profiles=[profile], - ) - compliance = Compliance_OCSF( - status=generate_json_ocsf_status(finding.status), - status_detail=finding.status_extended, - requirements=unroll_dict_to_list( - get_check_compliance( - finding, finding.check_metadata.Provider, output_options - ) - ), - ) - finding_output = Check_Output_JSON_OCSF( - finding=finding_ocsf, - resources=resources, - status_detail=finding.status_extended, - message=finding.status_extended, - severity=finding.check_metadata.Severity.capitalize(), - severity_id=generate_json_ocsf_severity_id(finding.check_metadata.Severity), - status=generate_json_ocsf_status(finding.status), - status_id=generate_json_ocsf_status_id(finding.status), - compliance=compliance, - cloud=cloud, - time=outputs_unix_timestamp(output_options.unix_timestamp, timestamp), - metadata=metadata, - ) - return finding_output - except Exception as error: - logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - - -def generate_json_ocsf_status(status: str): - json_ocsf_status = "" - if status == "PASS": - json_ocsf_status = "Success" - elif status == "FAIL": - json_ocsf_status = "Failure" - elif status == "MUTED": - json_ocsf_status = "Other" - else: - json_ocsf_status = "Unknown" - - return json_ocsf_status - - -def generate_json_ocsf_status_id(status: str): - json_ocsf_status_id = 0 - if status == "PASS": - json_ocsf_status_id = 1 - elif status == "FAIL": - json_ocsf_status_id = 2 - elif status == "MUTED": - json_ocsf_status_id = 99 - else: - json_ocsf_status_id = 0 - - return json_ocsf_status_id - - -def generate_json_ocsf_severity_id(severity: str): - json_ocsf_severity_id = 0 - if severity == "low": - json_ocsf_severity_id = 2 - elif severity == "medium": - json_ocsf_severity_id = 3 - elif severity == "high": - json_ocsf_severity_id = 4 - elif severity == "critical": - json_ocsf_severity_id = 5 - - return json_ocsf_severity_id - - -def close_json(output_filename, output_directory, mode): - """close_json closes the output JSON file replacing the last comma with ]""" - try: - suffix = json_file_suffix - if mode == "json-asff": - suffix = json_asff_file_suffix - elif mode == "json-ocsf": - suffix = json_ocsf_file_suffix - filename = f"{output_directory}/{output_filename}{suffix}" - # Close JSON file if exists - if os.path.isfile(filename): - file_descriptor = open_file( - filename, - "a", - ) - # Replace last comma for square bracket if not empty - if file_descriptor.tell() > 0: - if file_descriptor.tell() != 1: - file_descriptor.seek(file_descriptor.tell() - 1, os.SEEK_SET) - file_descriptor.truncate() - file_descriptor.write("]") - file_descriptor.close() - except Exception as error: - logger.critical( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}] -- {error}" - ) - sys.exit(1) diff --git a/prowler/lib/outputs/json/__init__.py b/prowler/lib/outputs/json/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/lib/outputs/json/json.py b/prowler/lib/outputs/json/json.py new file mode 100644 index 0000000000..1d1184f298 --- /dev/null +++ b/prowler/lib/outputs/json/json.py @@ -0,0 +1,39 @@ +import os +import sys + +from prowler.config.config import ( + json_asff_file_suffix, + json_file_suffix, + json_ocsf_file_suffix, +) +from prowler.lib.logger import logger +from prowler.lib.utils.utils import open_file + + +def close_json(output_filename, output_directory, mode): + """close_json closes the output JSON file replacing the last comma with ]""" + try: + suffix = json_file_suffix + if mode == "json-asff": + suffix = json_asff_file_suffix + elif mode == "json-ocsf": + suffix = json_ocsf_file_suffix + filename = f"{output_directory}/{output_filename}{suffix}" + # Close JSON file if exists + if os.path.isfile(filename): + file_descriptor = open_file( + filename, + "a", + ) + # Replace last comma for square bracket if not empty + if file_descriptor.tell() > 0: + if file_descriptor.tell() != 1: + file_descriptor.seek(file_descriptor.tell() - 1, os.SEEK_SET) + file_descriptor.truncate() + file_descriptor.write("]") + file_descriptor.close() + except Exception as error: + logger.critical( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}] -- {error}" + ) + sys.exit(1) diff --git a/prowler/lib/outputs/json_asff/__init__.py b/prowler/lib/outputs/json_asff/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/lib/outputs/json_asff/json_asff.py b/prowler/lib/outputs/json_asff/json_asff.py new file mode 100644 index 0000000000..956f970992 --- /dev/null +++ b/prowler/lib/outputs/json_asff/json_asff.py @@ -0,0 +1,121 @@ +from prowler.config.config import prowler_version, timestamp_utc +from prowler.lib.logger import logger +from prowler.lib.outputs.compliance.compliance import get_check_compliance +from prowler.lib.outputs.json_asff.models import ( + Compliance, + ProductFields, + Resource, + Severity, +) +from prowler.lib.utils.utils import hash_sha512 + + +def generate_json_asff_status(status: str) -> str: + json_asff_status = "" + if status == "PASS": + json_asff_status = "PASSED" + elif status == "FAIL": + json_asff_status = "FAILED" + elif status == "MUTED": + json_asff_status = "MUTED" + else: + json_asff_status = "NOT_AVAILABLE" + + return json_asff_status + + +def generate_json_asff_resource_tags(tags): + try: + resource_tags = {} + if tags and tags != [None]: + for tag in tags: + if "Key" in tag and "Value" in tag: + resource_tags[tag["Key"]] = tag["Value"] + else: + resource_tags.update(tag) + if len(resource_tags) == 0: + return None + else: + return None + return resource_tags + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + +def fill_json_asff(finding_output, provider, finding, output_options): + try: + # Check if there are no resources in the finding + if finding.resource_arn == "": + if finding.resource_id == "": + finding.resource_id = "NONE_PROVIDED" + finding.resource_arn = finding.resource_id + # The following line cannot be changed because it is the format we use to generate unique findings for AWS Security Hub + # If changed some findings could be lost because the unique identifier will be different + # TODO: get this from the provider output + finding_output.Id = f"prowler-{finding.check_metadata.CheckID}-{provider.identity.account}-{finding.region}-{hash_sha512(finding.resource_id)}" + finding_output.ProductArn = f"arn:{provider.identity.partition}:securityhub:{finding.region}::product/prowler/prowler" + finding_output.ProductFields = ProductFields( + ProviderVersion=prowler_version, ProwlerResourceName=finding.resource_arn + ) + finding_output.GeneratorId = "prowler-" + finding.check_metadata.CheckID + finding_output.AwsAccountId = provider.identity.account + finding_output.Types = finding.check_metadata.CheckType + finding_output.FirstObservedAt = finding_output.UpdatedAt = ( + finding_output.CreatedAt + ) = timestamp_utc.strftime("%Y-%m-%dT%H:%M:%SZ") + finding_output.Severity = Severity( + Label=finding.check_metadata.Severity.upper() + ) + finding_output.Title = finding.check_metadata.CheckTitle + # Description should NOT be longer than 1024 characters + finding_output.Description = ( + (finding.status_extended[:1000] + "...") + if len(finding.status_extended) > 1000 + else finding.status_extended + ) + resource_tags = generate_json_asff_resource_tags(finding.resource_tags) + finding_output.Resources = [ + Resource( + Id=finding.resource_arn, + Type=finding.check_metadata.ResourceType, + Partition=provider.identity.partition, + Region=finding.region, + Tags=resource_tags, + ) + ] + # Iterate for each compliance framework + compliance_summary = [] + associated_standards = [] + check_compliance = get_check_compliance(finding, "aws", output_options) + for key, value in check_compliance.items(): + if ( + len(associated_standards) < 20 + ): # AssociatedStandards should NOT have more than 20 items + associated_standards.append({"StandardsId": key}) + item = f"{key} {' '.join(value)}" + if len(item) > 64: + item = item[0:63] + compliance_summary.append(item) + + # Ensures finding_status matches allowed values in ASFF + finding_status = generate_json_asff_status(finding.status) + + finding_output.Compliance = Compliance( + Status=finding_status, + AssociatedStandards=associated_standards, + RelatedRequirements=compliance_summary, + ) + # Fill Recommendation Url if it is blank + if not finding.check_metadata.Remediation.Recommendation.Url: + finding.check_metadata.Remediation.Recommendation.Url = "https://docs.aws.amazon.com/securityhub/latest/userguide/what-is-securityhub.html" + finding_output.Remediation = { + "Recommendation": finding.check_metadata.Remediation.Recommendation + } + + return finding_output + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) diff --git a/prowler/lib/outputs/json_asff/models.py b/prowler/lib/outputs/json_asff/models.py new file mode 100644 index 0000000000..56738d611d --- /dev/null +++ b/prowler/lib/outputs/json_asff/models.py @@ -0,0 +1,51 @@ +from typing import Optional + +from pydantic import BaseModel + + +class ProductFields(BaseModel): + ProviderName: str = "Prowler" + ProviderVersion: str + ProwlerResourceName: str + + +class Severity(BaseModel): + Label: str + + +class Resource(BaseModel): + Type: str + Id: str + Partition: str + Region: str + Tags: Optional[dict] + + +class Compliance(BaseModel): + Status: str + RelatedRequirements: list[str] + AssociatedStandards: list[dict] + + +class Check_Output_JSON_ASFF(BaseModel): + """ + Check_Output_JSON_ASFF generates a finding's output in JSON ASFF format: https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-syntax.html + """ + + SchemaVersion: str = "2018-10-08" + Id: str = "" + ProductArn: str = "" + RecordState: str = "ACTIVE" + ProductFields: ProductFields = None # type: ignore + GeneratorId: str = "" + AwsAccountId: str = "" + Types: list[str] = None + FirstObservedAt: str = "" + UpdatedAt: str = "" + CreatedAt: str = "" + Severity: Severity = None # type: ignore + Title: str = "" + Description: str = "" + Resources: list[Resource] = None + Compliance: Compliance = None # type: ignore + Remediation: dict = None diff --git a/prowler/lib/outputs/json_ocsf/__init__.py b/prowler/lib/outputs/json_ocsf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/lib/outputs/json_ocsf/json_ocsf.py b/prowler/lib/outputs/json_ocsf/json_ocsf.py new file mode 100644 index 0000000000..18645f07aa --- /dev/null +++ b/prowler/lib/outputs/json_ocsf/json_ocsf.py @@ -0,0 +1,143 @@ +from py_ocsf_models.events.base_event import SeverityID, StatusID +from py_ocsf_models.events.findings.detection_finding import DetectionFinding +from py_ocsf_models.events.findings.detection_finding import ( + TypeID as DetectionFindingTypeID, +) +from py_ocsf_models.events.findings.finding import ActivityID, FindingInformation +from py_ocsf_models.objects.account import Account, TypeID +from py_ocsf_models.objects.cloud import Cloud +from py_ocsf_models.objects.container import Container +from py_ocsf_models.objects.group import Group +from py_ocsf_models.objects.metadata import Metadata +from py_ocsf_models.objects.organization import Organization +from py_ocsf_models.objects.product import Product +from py_ocsf_models.objects.remediation import Remediation +from py_ocsf_models.objects.resource_details import ResourceDetails + +from prowler.lib.logger import logger +from prowler.lib.outputs.common_models import FindingOutput + + +def get_account_type_id_by_provider(provider: str) -> TypeID: + type_id = TypeID.Other + if provider == "aws": + type_id = TypeID.AWS_Account + elif provider == "azure": + type_id = TypeID.Azure_AD_Account + elif provider == "gcp": + type_id = TypeID.GCP_Account + return type_id + + +def get_finding_status_id(status: str, muted: bool) -> StatusID: + status_id = StatusID.Other + if status == "FAIL": + status_id = StatusID.New + if muted: + status_id = StatusID.Suppressed + return status_id + + +def fill_json_ocsf(finding_output: FindingOutput) -> DetectionFinding: + try: + finding_activity = ActivityID.Create + cloud_account_type = get_account_type_id_by_provider(finding_output.provider) + finding_severity = getattr( + SeverityID, finding_output.severity.capitalize(), SeverityID.Unknown + ) + finding_status = get_finding_status_id( + finding_output.status, finding_output.muted + ) + + detection_finding = DetectionFinding( + activity_id=finding_activity.value, + activity_name=finding_activity.name, + finding_info=FindingInformation( + created_time=finding_output.timestamp, + desc=finding_output.description, + title=finding_output.check_title, + uid=finding_output.finding_uid, + product_uid="prowler", + ), + event_time=finding_output.timestamp, + remediation=Remediation( + desc=finding_output.remediation_recommendation_text, + references=list( + filter( + None, + [ + finding_output.remediation_code_nativeiac, + finding_output.remediation_code_terraform, + finding_output.remediation_code_cli, + finding_output.remediation_code_other, + finding_output.remediation_recommendation_url, + ], + ) + ), + ), + severity_id=finding_severity.value, + severity=finding_severity.name, + status_id=finding_status.value, + status=finding_status.name, + status_code=finding_output.status, + status_detail=finding_output.status_extended, + resources=[ + ResourceDetails( + # TODO: Check labels for other providers + labels=( + finding_output.resource_tags.split(",") + if finding_output.resource_tags + else [] + ), + name=finding_output.resource_name, + uid=finding_output.resource_uid, + group=Group(name=finding_output.service_name), + type=finding_output.resource_type, + cloud_partition=finding_output.partition, + region=finding_output.region, + ) + ], + metadata=Metadata( + product=Product( + name="Prowler", + vendor_name="Prowler", + version=finding_output.prowler_version, + ), + ), + type_uid=DetectionFindingTypeID.Create, + type_name=DetectionFindingTypeID.Create.name, + ) + + if finding_output.provider == "kubernetes": + detection_finding.container = ( + Container( + name=finding_output.resource_name, + uid=finding_output.resource_uid, + ), + ) + # TODO: Get the PID of the namespace (we only have the name of the namespace) + # detection_finding.namespace_pid=, + else: + detection_finding.cloud = ( + Cloud( + account=Account( + name=finding_output.account_name, + type_id=cloud_account_type.value, + type=cloud_account_type.name, + uid=finding_output.account_uid, + ), + org=Organization( + uid=finding_output.account_organization_uid, + name=finding_output.account_organization_name, + ), + provider=finding_output.provider, + region=finding_output.region, + ), + ) + + return detection_finding + + except Exception as error: + logger.error( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) diff --git a/prowler/lib/outputs/models.py b/prowler/lib/outputs/models.py deleted file mode 100644 index d6ef8363a8..0000000000 --- a/prowler/lib/outputs/models.py +++ /dev/null @@ -1,469 +0,0 @@ -import sys -from datetime import datetime -from typing import List, Literal, Optional - -from pydantic import BaseModel - -from prowler.config.config import prowler_version -from prowler.lib.logger import logger -from prowler.lib.utils.utils import outputs_unix_timestamp -from prowler.providers.aws.models import AWSOrganizationsInfo - - -def get_check_compliance(finding, provider_type, output_options) -> dict: - """get_check_compliance returns a map with the compliance framework as key and the requirements where the finding's check is present. - - Example: - - { - "CIS-1.4": ["2.1.3"], - "CIS-1.5": ["2.1.3"], - } - """ - try: - check_compliance = {} - # We have to retrieve all the check's compliance requirements - if finding.check_metadata.CheckID in output_options.bulk_checks_metadata: - for compliance in output_options.bulk_checks_metadata[ - finding.check_metadata.CheckID - ].Compliance: - compliance_fw = compliance.Framework - if compliance.Version: - compliance_fw = f"{compliance_fw}-{compliance.Version}" - if compliance.Provider == provider_type.upper(): - if compliance_fw not in check_compliance: - check_compliance[compliance_fw] = [] - for requirement in compliance.Requirements: - check_compliance[compliance_fw].append(requirement.Id) - return check_compliance - except Exception as error: - logger.critical( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}] -- {error}" - ) - sys.exit(1) - - -def unroll_list(listed_items: list): - unrolled_items = "" - separator = "|" - if listed_items: - for item in listed_items: - if not unrolled_items: - unrolled_items = f"{item}" - else: - unrolled_items = f"{unrolled_items} {separator} {item}" - - return unrolled_items - - -def unroll_tags(tags: list): - unrolled_items = "" - separator = "|" - if tags and tags != [{}] and tags != [None]: - for item in tags: - # Check if there are tags in list - if isinstance(item, dict): - for key, value in item.items(): - if not unrolled_items: - # Check the pattern of tags (Key:Value or Key:key/Value:value) - if "Key" != key and "Value" != key: - unrolled_items = f"{key}={value}" - else: - if "Key" == key: - unrolled_items = f"{value}=" - else: - unrolled_items = f"{value}" - else: - if "Key" != key and "Value" != key: - unrolled_items = ( - f"{unrolled_items} {separator} {key}={value}" - ) - else: - if "Key" == key: - unrolled_items = ( - f"{unrolled_items} {separator} {value}=" - ) - else: - unrolled_items = f"{unrolled_items}{value}" - elif not unrolled_items: - unrolled_items = f"{item}" - else: - unrolled_items = f"{unrolled_items} {separator} {item}" - - return unrolled_items - - -def unroll_dict(dict: dict): - unrolled_items = "" - separator = "|" - for key, value in dict.items(): - if isinstance(value, list): - value = ", ".join(value) - if not unrolled_items: - unrolled_items = f"{key}: {value}" - else: - unrolled_items = f"{unrolled_items} {separator} {key}: {value}" - - return unrolled_items - - -def unroll_dict_to_list(dict: dict): - dict_list = [] - for key, value in dict.items(): - if isinstance(value, list): - value = ", ".join(value) - dict_list.append(f"{key}: {value}") - else: - dict_list.append(f"{key}: {value}") - - return dict_list - - -def parse_json_tags(tags: list): - dict_tags = {} - if tags and tags != [{}] and tags != [None]: - for tag in tags: - if "Key" in tag and "Value" in tag: - dict_tags[tag["Key"]] = tag["Value"] - else: - dict_tags.update(tag) - - return dict_tags - - -class Check_Output_MITRE_ATTACK(BaseModel): - """ - Check_Output_MITRE_ATTACK generates a finding's output in CSV MITRE ATTACK format. - """ - - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Id: str - Requirements_Name: str - Requirements_Description: str - Requirements_Tactics: str - Requirements_SubTechniques: str - Requirements_Platforms: str - Requirements_TechniqueURL: str - Requirements_Attributes_AWSServices: str - Requirements_Attributes_Categories: str - Requirements_Attributes_Values: str - Requirements_Attributes_Comments: str - Status: str - StatusExtended: str - ResourceId: str - CheckId: str - - -class Check_Output_CSV_ENS_RD2022(BaseModel): - """ - Check_Output_CSV_ENS_RD2022 generates a finding's output in CSV ENS RD2022 format. - """ - - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Id: str - Requirements_Description: str - Requirements_Attributes_IdGrupoControl: str - Requirements_Attributes_Marco: str - Requirements_Attributes_Categoria: str - Requirements_Attributes_DescripcionControl: str - Requirements_Attributes_Nivel: str - Requirements_Attributes_Tipo: str - Requirements_Attributes_Dimensiones: str - Status: str - StatusExtended: str - ResourceId: str - CheckId: str - - -class Check_Output_CSV_AWS_CIS(BaseModel): - """ - Check_Output_CSV_CIS generates a finding's output in CSV CIS format. - """ - - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Id: str - Requirements_Description: str - Requirements_Attributes_Section: str - Requirements_Attributes_Profile: str - Requirements_Attributes_AssessmentStatus: str - Requirements_Attributes_Description: str - Requirements_Attributes_RationaleStatement: str - Requirements_Attributes_ImpactStatement: str - Requirements_Attributes_RemediationProcedure: str - Requirements_Attributes_AuditProcedure: str - Requirements_Attributes_AdditionalInformation: str - Requirements_Attributes_References: str - Status: str - StatusExtended: str - ResourceId: str - CheckId: str - - -class Check_Output_CSV_GCP_CIS(BaseModel): - """ - Check_Output_CSV_CIS generates a finding's output in CSV CIS format. - """ - - Provider: str - Description: str - ProjectId: str - Location: str - AssessmentDate: str - Requirements_Id: str - Requirements_Description: str - Requirements_Attributes_Section: str - Requirements_Attributes_Profile: str - Requirements_Attributes_AssessmentStatus: str - Requirements_Attributes_Description: str - Requirements_Attributes_RationaleStatement: str - Requirements_Attributes_ImpactStatement: str - Requirements_Attributes_RemediationProcedure: str - Requirements_Attributes_AuditProcedure: str - Requirements_Attributes_AdditionalInformation: str - Requirements_Attributes_References: str - Status: str - StatusExtended: str - ResourceId: str - ResourceName: str - CheckId: str - - -class Check_Output_CSV_Generic_Compliance(BaseModel): - """ - Check_Output_CSV_Generic_Compliance generates a finding's output in CSV Generic Compliance format. - """ - - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Id: str - Requirements_Description: str - Requirements_Attributes_Section: Optional[str] - Requirements_Attributes_SubSection: Optional[str] - Requirements_Attributes_SubGroup: Optional[str] - Requirements_Attributes_Service: Optional[str] - Requirements_Attributes_Type: Optional[str] - Status: str - StatusExtended: str - ResourceId: str - CheckId: str - - -class Check_Output_CSV_AWS_Well_Architected(BaseModel): - """ - Check_Output_CSV_AWS_Well_Architected generates a finding's output in CSV AWS Well Architected Compliance format. - """ - - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Attributes_Name: str - Requirements_Attributes_WellArchitectedQuestionId: str - Requirements_Attributes_WellArchitectedPracticeId: str - Requirements_Attributes_Section: str - Requirements_Attributes_SubSection: Optional[str] - Requirements_Attributes_LevelOfRisk: str - Requirements_Attributes_AssessmentMethod: str - Requirements_Attributes_Description: str - Requirements_Attributes_ImplementationGuidanceUrl: str - Status: str - StatusExtended: str - ResourceId: str - CheckId: str - - -class Check_Output_CSV_AWS_ISO27001_2013(BaseModel): - """ - Check_Output_CSV_AWS_ISO27001_2013 generates a finding's output in CSV AWS ISO27001 Compliance format. - """ - - Provider: str - Description: str - AccountId: str - Region: str - AssessmentDate: str - Requirements_Attributes_Category: str - Requirements_Attributes_Objetive_ID: str - Requirements_Attributes_Objetive_Name: str - Requirements_Attributes_Check_Summary: str - Status: str - StatusExtended: str - ResourceId: str - CheckId: str - - -# JSON ASFF Output -class ProductFields(BaseModel): - ProviderName: str = "Prowler" - ProviderVersion: str - ProwlerResourceName: str - - -class Severity(BaseModel): - Label: str - - -class Resource(BaseModel): - Type: str - Id: str - Partition: str - Region: str - Tags: Optional[dict] - - -class Compliance(BaseModel): - Status: str - RelatedRequirements: List[str] - AssociatedStandards: List[dict] - - -class Check_Output_JSON_ASFF(BaseModel): - """ - Check_Output_JSON_ASFF generates a finding's output in JSON ASFF format: https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format-syntax.html - """ - - SchemaVersion: str = "2018-10-08" - Id: str = "" - ProductArn: str = "" - RecordState: str = "ACTIVE" - ProductFields: ProductFields = None - GeneratorId: str = "" - AwsAccountId: str = "" - Types: List[str] = None - FirstObservedAt: str = "" - UpdatedAt: str = "" - CreatedAt: str = "" - Severity: Severity = None - Title: str = "" - Description: str = "" - Resources: List[Resource] = None - Compliance: Compliance = None - Remediation: dict = None - - -# JSON OCSF -class Remediation_OCSF(BaseModel): - kb_articles: List[str] - desc: str - - -class Finding(BaseModel): - title: str - desc: str - supporting_data: dict - remediation: Remediation_OCSF - types: List[str] - src_url: str - uid: str - related_events: List[str] - - -class Group(BaseModel): - name: str - - -class Resources(BaseModel): - group: Group - region: str - name: str - uid: str - labels: list - type: str - details: str - - -class Compliance_OCSF(BaseModel): - status: str - requirements: List[str] - status_detail: str - - -class Account(BaseModel): - name: str - uid: str - - -class Organization(BaseModel): - uid: str - name: str - - -class Cloud(BaseModel): - account: Optional[Account] - region: str - org: Optional[Organization] - provider: str - project_uid: str - - -class Feature(BaseModel): - name: str - uid: str - version: str = prowler_version - - -class Product(BaseModel): - language: str = "en" - name: str = "Prowler" - version: str = prowler_version - vendor_name: str = "Prowler/ProwlerPro" - feature: Feature - - -class Metadata(BaseModel): - original_time: str - profiles: List[str] - product: Product - version: str = "1.0.0-rc.3" - - -class Check_Output_JSON_OCSF(BaseModel): - """ - Check_Output_JSON_OCSF generates a finding's output in JSON OCSF format. - https://schema.ocsf.io/1.0.0-rc.3/classes/security_finding - """ - - finding: Finding - resources: List[Resources] - status_detail: str - compliance: Compliance_OCSF - message: str - severity_id: Literal[0, 1, 2, 3, 4, 5, 6, 99] - severity: Literal[ - "Informational", "Low", "Medium", "High", "Critical", "Fatal", "Other" - ] - cloud: Cloud - time: datetime - metadata: Metadata - state_id: int = 0 - state: str = "New" - status_id: Literal[0, 1, 2, 99] - status: Literal["Unknown", "Success", "Failure", "Other"] - type_uid: int = 200101 - type_name: str = "Security Finding: Create" - impact_id: int = 0 - impact: str = "Unknown" - confidence_id: int = 0 - confidence: str = "Unknown" - activity_id: int = 1 - activity_name: str = "Create" - category_uid: int = 2 - category_name: str = "Findings" - class_uid: int = 2001 - class_name: str = "Security Finding" diff --git a/prowler/lib/outputs/outputs.py b/prowler/lib/outputs/outputs.py index 7fa0b9bf6a..a1f7989698 100644 --- a/prowler/lib/outputs/outputs.py +++ b/prowler/lib/outputs/outputs.py @@ -5,24 +5,23 @@ from colorama import Fore, Style from prowler.config.config import available_compliance_frameworks, orange_color from prowler.lib.logger import logger +from prowler.lib.outputs.common import ( + fill_common_finding_data, + generate_provider_output, + get_provider_data_mapping, +) +from prowler.lib.outputs.common_models import FindingOutput from prowler.lib.outputs.compliance.compliance import ( add_manual_controls, fill_compliance, -) -from prowler.lib.outputs.csv.csv import ( - fill_common_data_csv, - generate_csv_fields, - generate_provider_output_csv, - get_provider_data_mapping, -) -from prowler.lib.outputs.csv.models import CSVRow -from prowler.lib.outputs.file_descriptors import fill_file_descriptors -from prowler.lib.outputs.json import fill_json_asff, fill_json_ocsf -from prowler.lib.outputs.models import ( - Check_Output_JSON_ASFF, get_check_compliance, - unroll_dict, ) +from prowler.lib.outputs.csv.csv import generate_csv_fields +from prowler.lib.outputs.file_descriptors import fill_file_descriptors +from prowler.lib.outputs.json_asff.json_asff import fill_json_asff +from prowler.lib.outputs.json_asff.models import Check_Output_JSON_ASFF +from prowler.lib.outputs.json_ocsf.json_ocsf import fill_json_ocsf +from prowler.lib.outputs.utils import unroll_dict def stdout_report(finding, color, verbose, status): @@ -100,6 +99,7 @@ def report(check_findings, provider): # AWS specific outputs if finding.check_metadata.Provider == "aws": if "json-asff" in file_descriptors: + # Initialize this field using the class within fill_json_asff not here finding_output = Check_Output_JSON_ASFF() fill_json_asff( finding_output, provider, finding, output_options @@ -112,44 +112,37 @@ def report(check_findings, provider): ) file_descriptors["json-asff"].write(",") + # Common Output Data + provider_data_mapping = get_provider_data_mapping(provider) + common_finding_data = fill_common_finding_data( + finding, output_options.unix_timestamp + ) + csv_data = {} + csv_data.update(provider_data_mapping) + csv_data.update(common_finding_data) + csv_data["compliance"] = unroll_dict( + get_check_compliance(finding, provider.type, output_options) + ) + finding_output = generate_provider_output( + provider, finding, csv_data + ) + # CSV if "csv" in file_descriptors: - provider_data = get_provider_data_mapping(provider) - common_data = fill_common_data_csv( - finding, output_options.unix_timestamp - ) - compliance_data = unroll_dict( - get_check_compliance( - finding, provider.type, output_options - ) - ) - csv_data = {} - csv_data.update(provider_data) - csv_data.update(common_data) - csv_data["compliance"] = compliance_data - csv_writer = DictWriter( file_descriptors["csv"], - fieldnames=generate_csv_fields(CSVRow), + fieldnames=generate_csv_fields(FindingOutput), delimiter=";", ) - finding_output = generate_provider_output_csv( - provider, finding, csv_data - ) - csv_writer.writerow(finding_output.dict()) + # JSON if "json-ocsf" in file_descriptors: - finding_output = fill_json_ocsf( - provider, finding, output_options - ) - - json.dump( - finding_output.dict(), - file_descriptors["json-ocsf"], - indent=4, - default=str, + detection_finding = fill_json_ocsf(finding_output) + # print(file) + file_descriptors["json-ocsf"].write( + detection_finding.json(exclude_none=True, indent=4) ) file_descriptors["json-ocsf"].write(",") diff --git a/prowler/lib/outputs/utils.py b/prowler/lib/outputs/utils.py new file mode 100644 index 0000000000..e55c1fcbf6 --- /dev/null +++ b/prowler/lib/outputs/utils.py @@ -0,0 +1,86 @@ +def unroll_list(listed_items: list): + unrolled_items = "" + separator = "|" + if listed_items: + for item in listed_items: + if not unrolled_items: + unrolled_items = f"{item}" + else: + unrolled_items = f"{unrolled_items} {separator} {item}" + + return unrolled_items + + +def unroll_tags(tags: list): + unrolled_items = "" + separator = "|" + if tags and tags != [{}] and tags != [None]: + for item in tags: + # Check if there are tags in list + if isinstance(item, dict): + for key, value in item.items(): + if not unrolled_items: + # Check the pattern of tags (Key:Value or Key:key/Value:value) + if "Key" != key and "Value" != key: + unrolled_items = f"{key}={value}" + else: + if "Key" == key: + unrolled_items = f"{value}=" + else: + unrolled_items = f"{value}" + else: + if "Key" != key and "Value" != key: + unrolled_items = ( + f"{unrolled_items} {separator} {key}={value}" + ) + else: + if "Key" == key: + unrolled_items = ( + f"{unrolled_items} {separator} {value}=" + ) + else: + unrolled_items = f"{unrolled_items}{value}" + elif not unrolled_items: + unrolled_items = f"{item}" + else: + unrolled_items = f"{unrolled_items} {separator} {item}" + + return unrolled_items + + +def unroll_dict(dict: dict): + unrolled_items = "" + separator = "|" + for key, value in dict.items(): + if isinstance(value, list): + value = ", ".join(value) + if not unrolled_items: + unrolled_items = f"{key}: {value}" + else: + unrolled_items = f"{unrolled_items} {separator} {key}: {value}" + + return unrolled_items + + +def unroll_dict_to_list(dict: dict): + dict_list = [] + for key, value in dict.items(): + if isinstance(value, list): + value = ", ".join(value) + dict_list.append(f"{key}: {value}") + else: + dict_list.append(f"{key}: {value}") + + return dict_list + + +def parse_json_tags(tags: list): + dict_tags = {} + if tags and tags != [{}] and tags != [None]: + for tag in tags: + if "Key" in tag and "Value" in tag: + dict_tags[tag["Key"]] = tag["Value"] + else: + dict_tags.update(tag) + + return dict_tags diff --git a/prowler/providers/aws/aws_provider.py b/prowler/providers/aws/aws_provider.py index 9d91059644..e5e81b5f9c 100644 --- a/prowler/providers/aws/aws_provider.py +++ b/prowler/providers/aws/aws_provider.py @@ -290,11 +290,11 @@ class AwsProvider(Provider): "auth_method": "identity.profile", "provider": "type", "account_uid": "identity.account", - "account_name": "organizations_metadata.account_details_name", - "account_email": "organizations_metadata.account_details_email", - "account_organization_uid": "organizations_metadata.account_details_arn", - "account_organization": "organizations_metadata.account_details_org", - "account_tags": "organizations_metadata.account_details_tags", + "account_name": "organizations_metadata.account_name", + "account_email": "organizations_metadata.account_email", + "account_organization_uid": "organizations_metadata.organization_arn", + "account_organization_name": "organizations_metadata.organization_id", + "account_tags": "organizations_metadata.account_tags", "partition": "identity.partition", } diff --git a/prowler/providers/aws/lib/arn/error.py b/prowler/providers/aws/lib/arn/error.py index 13e784214e..31100ee1e0 100644 --- a/prowler/providers/aws/lib/arn/error.py +++ b/prowler/providers/aws/lib/arn/error.py @@ -1,5 +1,5 @@ class RoleArnParsingFailedMissingFields(Exception): - # The ARN contains a numberof fields different than six separated by :" + # The ARN contains a number of fields different than six separated by :" def __init__(self): self.message = "The assumed role ARN contains an invalid number of fields separated by : or it does not start by arn, please input a valid ARN" super().__init__(self.message) diff --git a/prowler/providers/aws/lib/mutelist/mutelist.py b/prowler/providers/aws/lib/mutelist/mutelist.py index 76114db6c4..a69615a4e8 100644 --- a/prowler/providers/aws/lib/mutelist/mutelist.py +++ b/prowler/providers/aws/lib/mutelist/mutelist.py @@ -8,7 +8,7 @@ from boto3.dynamodb.conditions import Attr from schema import Optional, Schema from prowler.lib.logger import logger -from prowler.lib.outputs.models import unroll_tags +from prowler.lib.outputs.utils import unroll_tags mutelist_schema = Schema( { diff --git a/prowler/providers/aws/lib/organizations/organizations.py b/prowler/providers/aws/lib/organizations/organizations.py index c34196167d..12748f66c3 100644 --- a/prowler/providers/aws/lib/organizations/organizations.py +++ b/prowler/providers/aws/lib/organizations/organizations.py @@ -1,6 +1,7 @@ from boto3 import session from prowler.lib.logger import logger +from prowler.providers.aws.lib.arn.models import ARN from prowler.providers.aws.models import AWSOrganizationsInfo @@ -34,14 +35,19 @@ def parse_organizations_metadata(metadata: dict, tags: dict) -> AWSOrganizations account_details_tags += f"{tag['Key']}:{tag['Value']}," account_details = metadata.get("Account", {}) - organizations_info = AWSOrganizationsInfo( - account_details_email=account_details.get("Email", ""), - account_details_name=account_details.get("Name", ""), - account_details_arn=account_details.get("Arn", ""), - account_details_org=account_details.get("Arn", "").split("/")[1], - account_details_tags=account_details_tags.rstrip(","), + + aws_account_arn = ARN(account_details.get("Arn", "")) + aws_organization_id = aws_account_arn.resource.split("/")[0] + aws_organization_arn = f"arn:{aws_account_arn.partition}:organizations::{aws_account_arn.account_id}:organization/{aws_organization_id}" + + return AWSOrganizationsInfo( + account_email=account_details.get("Email", ""), + account_name=account_details.get("Name", ""), + organization_account_arn=aws_account_arn.arn, + organization_arn=aws_organization_arn, + organization_id=aws_organization_id, + account_tags=account_details_tags.rstrip(","), ) - return organizations_info except Exception as error: logger.warning( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" diff --git a/prowler/providers/aws/lib/quick_inventory/quick_inventory.py b/prowler/providers/aws/lib/quick_inventory/quick_inventory.py index 6bc15ea46f..db7ad39e6b 100644 --- a/prowler/providers/aws/lib/quick_inventory/quick_inventory.py +++ b/prowler/providers/aws/lib/quick_inventory/quick_inventory.py @@ -284,7 +284,7 @@ def create_output(resources: list, audit_info: AWS_Audit_Info, args): print(f" - JSON: {args.output_directory}/{output_file + json_file_suffix}") # Send output to S3 if needed (-B / -D) - for mode in ["json", "csv"]: + for mode in ["csv"]: if args.output_bucket or args.output_bucket_no_assume: # Check if -B was input if args.output_bucket: diff --git a/prowler/providers/aws/lib/security_hub/security_hub.py b/prowler/providers/aws/lib/security_hub/security_hub.py index 77f58e3875..94e83ee575 100644 --- a/prowler/providers/aws/lib/security_hub/security_hub.py +++ b/prowler/providers/aws/lib/security_hub/security_hub.py @@ -3,8 +3,8 @@ from botocore.client import ClientError from prowler.config.config import timestamp_utc from prowler.lib.logger import logger -from prowler.lib.outputs.json import fill_json_asff -from prowler.lib.outputs.models import Check_Output_JSON_ASFF +from prowler.lib.outputs.json_asff.json_asff import fill_json_asff +from prowler.lib.outputs.json_asff.models import Check_Output_JSON_ASFF SECURITY_HUB_INTEGRATION_NAME = "prowler/prowler" SECURITY_HUB_MAX_BATCH = 100 diff --git a/prowler/providers/aws/models.py b/prowler/providers/aws/models.py index 64cc0c9eba..c027b493bf 100644 --- a/prowler/providers/aws/models.py +++ b/prowler/providers/aws/models.py @@ -11,11 +11,12 @@ from prowler.providers.common.models import ProviderOutputOptions @dataclass class AWSOrganizationsInfo: - account_details_email: str - account_details_name: str - account_details_arn: str - account_details_org: str - account_details_tags: str + account_email: str + account_name: str + organization_account_arn: str + organization_arn: str + organization_id: str + account_tags: str @dataclass diff --git a/prowler/providers/azure/azure_provider.py b/prowler/providers/azure/azure_provider.py index 92b4adf27a..71fd62e1ba 100644 --- a/prowler/providers/azure/azure_provider.py +++ b/prowler/providers/azure/azure_provider.py @@ -118,7 +118,7 @@ class AzureProvider(Provider): # TODO: check the tenant_ids # TODO: we have to get the account organization, the tenant is not that "account_organization_uid": "identity.tenant_ids", - "account_organization": "identity.tenant_domain", + "account_organization_name": "identity.tenant_domain", # TODO: pending to get the subscription tags # "account_tags": "organizations_metadata.account_details_tags", "partition": "region_config.name", diff --git a/pyproject.toml b/pyproject.toml index 75684e78fb..d875d5f6fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ jsonschema = "4.21.1" kubernetes = "28.1.0" msgraph-sdk = "1.0.0" msrestazure = "0.6.4" +py-ocsf-models = "0.0.3" pydantic = "1.10.14" python = ">=3.9,<3.13" schema = "0.7.5" diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index d4dc44394d..b17d1dbc0e 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -41,10 +41,10 @@ class Test_Parser: command = [prowler_command] parsed = self.parser.parse(command) assert parsed.provider == provider - assert not parsed.quiet - assert len(parsed.output_modes) == 4 + assert not parsed.status + assert len(parsed.output_modes) == 2 assert "csv" in parsed.output_modes - + assert "json-ocsf" in parsed.output_modes assert not parsed.output_filename assert "output" in parsed.output_directory assert not parsed.verbose @@ -88,10 +88,10 @@ class Test_Parser: command = [prowler_command, provider] parsed = self.parser.parse(command) assert parsed.provider == provider - assert not parsed.quiet - assert len(parsed.output_modes) == 4 + assert not parsed.status + assert len(parsed.output_modes) == 2 assert "csv" in parsed.output_modes - + assert "json-ocsf" in parsed.output_modes assert not parsed.output_filename assert "output" in parsed.output_directory assert not parsed.verbose @@ -127,10 +127,10 @@ class Test_Parser: command = [prowler_command, provider] parsed = self.parser.parse(command) assert parsed.provider == provider - assert not parsed.quiet - assert len(parsed.output_modes) == 4 + assert not parsed.status + assert len(parsed.output_modes) == 2 assert "csv" in parsed.output_modes - + assert "json-ocsf" in parsed.output_modes assert not parsed.output_filename assert "output" in parsed.output_directory assert not parsed.verbose @@ -161,10 +161,10 @@ class Test_Parser: command = [prowler_command, provider] parsed = self.parser.parse(command) assert parsed.provider == provider - assert not parsed.quiet - assert len(parsed.output_modes) == 4 + assert not parsed.severity + assert len(parsed.output_modes) == 2 assert "csv" in parsed.output_modes - + assert "json-ocsf" in parsed.output_modes assert not parsed.output_filename assert "output" in parsed.output_directory assert not parsed.verbose @@ -188,7 +188,9 @@ class Test_Parser: assert not parsed.list_compliance assert not parsed.list_compliance_requirements assert not parsed.list_categories - assert not parsed.credentials_file + assert not parsed.kubeconfig_file + assert not parsed.context + assert not parsed.namespaces def test_root_parser_version_short(self): command = [prowler_command, "-v"] @@ -244,9 +246,9 @@ class Test_Parser: assert parsed.provider == "kubernetes" def test_root_parser_status(self): - command = [prowler_command, "--status"] + command = [prowler_command, "--status", "FAIL"] parsed = self.parser.parse(command) - assert parsed.status + assert parsed.status == ["FAIL"] def test_root_parser_exit_code_3_short(self): command = [prowler_command, "-z"] @@ -261,8 +263,9 @@ class Test_Parser: def test_root_parser_default_output_modes(self): command = [prowler_command] parsed = self.parser.parse(command) - assert len(parsed.output_modes) == 4 + assert len(parsed.output_modes) == 2 assert "csv" in parsed.output_modes + assert "json-ocsf" in parsed.output_modes def test_root_parser_output_modes_short(self): command = [prowler_command, "-M", "csv"] @@ -276,6 +279,18 @@ class Test_Parser: assert len(parsed.output_modes) == 1 assert "csv" in parsed.output_modes + def test_root_parser_output_modes_short_json_ocsf(self): + command = [prowler_command, "-M", "json-ocsf"] + parsed = self.parser.parse(command) + assert len(parsed.output_modes) == 1 + assert "json-ocsf" in parsed.output_modes + + def test_root_parser_output_modes_long_json_ocsf(self): + command = [prowler_command, "--output-modes", "json-ocsf"] + parsed = self.parser.parse(command) + assert len(parsed.output_modes) == 1 + assert "json-ocsf" in parsed.output_modes + def test_root_parser_output_filename_short(self): filename = "test_output.txt" command = [prowler_command, "-F", filename] @@ -1178,11 +1193,12 @@ class Test_Parser: def test_parser_kubernetes_auth_namespace(self): argument = "--namespaces" - namespaces = ["default", "kube-system"] - command = [prowler_command, "kubernetes", argument, namespaces] + namespace_1 = "default" + namespace_2 = "kube-system" + command = [prowler_command, "kubernetes", argument, namespace_1, namespace_2] parsed = self.parser.parse(command) assert parsed.provider == "kubernetes" - assert parsed.namespaces == namespaces + assert parsed.namespaces == [namespace_1, namespace_2] def test_validate_azure_region_valid_regions(self): expected_regions = [ diff --git a/tests/lib/outputs/outputs_test.py b/tests/lib/outputs/outputs_test.py index ac62bf6525..9b93741f5c 100644 --- a/tests/lib/outputs/outputs_test.py +++ b/tests/lib/outputs/outputs_test.py @@ -29,9 +29,6 @@ from prowler.lib.outputs.json import ( fill_json_ocsf, generate_json_asff_resource_tags, generate_json_asff_status, - generate_json_ocsf_severity_id, - generate_json_ocsf_status, - generate_json_ocsf_status_id, ) from prowler.lib.outputs.models import ( Account, @@ -1305,22 +1302,3 @@ class Test_Outputs: assert generate_json_asff_resource_tags( [{"Key": "key1", "Value": "value1"}] ) == {"key1": "value1"} - - def test_generate_json_ocsf_status(self): - assert generate_json_ocsf_status("PASS") == "Success" - assert generate_json_ocsf_status("FAIL") == "Failure" - assert generate_json_ocsf_status("MUTED") == "Other" - assert generate_json_ocsf_status("SOMETHING ELSE") == "Unknown" - - def test_generate_json_ocsf_status_id(self): - assert generate_json_ocsf_status_id("PASS") == 1 - assert generate_json_ocsf_status_id("FAIL") == 2 - assert generate_json_ocsf_status_id("MUTED") == 99 - assert generate_json_ocsf_status_id("SOMETHING ELSE") == 0 - - def test_generate_json_ocsf_severity_id(self): - assert generate_json_ocsf_severity_id("low") == 2 - assert generate_json_ocsf_severity_id("medium") == 3 - assert generate_json_ocsf_severity_id("high") == 4 - assert generate_json_ocsf_severity_id("critical") == 5 - assert generate_json_ocsf_severity_id("something else") == 0