diff --git a/docs/tutorials/quick-inventory.md b/docs/tutorials/quick-inventory.md index 6221446967..89ba813e44 100644 --- a/docs/tutorials/quick-inventory.md +++ b/docs/tutorials/quick-inventory.md @@ -6,7 +6,7 @@ Prowler allows you to execute a quick inventory to extract the number of resourc Currently, it is only available for AWS provider. -- You can use option `-i`/`--quick-inventory` to execute it: +- You can use option `-i`/`--inventory` to execute it: ```sh prowler -i ``` diff --git a/poetry.lock b/poetry.lock index f9bee54f00..a5d2ad9ebd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "about-time" @@ -4643,6 +4643,26 @@ files = [ {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] +[[package]] +name = "tqdm" +version = "4.66.5" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, + {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typer" version = "0.12.5" @@ -5060,4 +5080,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "6ab3e45b4316275b0c26103a5dff489527d927bb4d9e4ff6a847849cb106d837" +content-hash = "68fb9f6bc687d8be8324a80f955119d9934750cfd3ee3ffd6745c109e5334483" diff --git a/prowler/__main__.py b/prowler/__main__.py index 7a229112f3..68ad0b800c 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -71,8 +71,8 @@ from prowler.providers.aws.lib.s3.s3 import S3 from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub from prowler.providers.aws.models import AWSOutputOptions from prowler.providers.azure.models import AzureOutputOptions +from prowler.providers.common.inventory import run_prowler_inventory from prowler.providers.common.provider import Provider -from prowler.providers.common.quick_inventory import run_provider_quick_inventory from prowler.providers.gcp.models import GCPOutputOptions from prowler.providers.kubernetes.models import KubernetesOutputOptions @@ -257,11 +257,6 @@ def prowler(): args, bulk_checks_metadata, global_provider.identity ) - # Run the quick inventory for the provider if available - if hasattr(args, "quick_inventory") and args.quick_inventory: - run_provider_quick_inventory(global_provider, args) - sys.exit() - # Execute checks findings = [] @@ -688,6 +683,11 @@ def prowler(): if checks_folder: remove_custom_checks_module(checks_folder, provider) + # Run the quick inventory for the provider if available + if hasattr(args, "inventory") and args.inventory: + run_prowler_inventory(checks_to_execute, args.provider) + sys.exit() + # If there are failed findings exit code 3, except if -z is input if ( not args.ignore_exit_code_3 diff --git a/prowler/lib/cli/parser.py b/prowler/lib/cli/parser.py index 2309d8742c..9c62b6bbb8 100644 --- a/prowler/lib/cli/parser.py +++ b/prowler/lib/cli/parser.py @@ -378,3 +378,16 @@ Detailed documentation at https://docs.prowler.com action="store_true", help="Send a summary of the execution with a Slack APP in your channel. Environment variables SLACK_API_TOKEN and SLACK_CHANNEL_NAME are required (see more in https://docs.prowler.cloud/en/latest/tutorials/integrations/#slack).", ) + + def __init_inventory_parser__(self): + inventory_parser = self.common_providers_parser.add_argument_group("Inventory") + inventory_parser.add_argument( + "--inventory", + action="store_true", + help="Run Prowler in inventory mode", + ) + inventory_parser.add_argument( + "-i", + nargs="?", + help="Output folder path for the inventory", + ) diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index 8cac4b6a1a..cd6014d030 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -93,12 +93,12 @@ def init_parser(self): help="Send only Prowler failed findings to SecurityHub", ) # AWS Quick Inventory - aws_quick_inventory_subparser = aws_parser.add_argument_group("Quick Inventory") + aws_quick_inventory_subparser = aws_parser.add_argument_group("Inventory") aws_quick_inventory_subparser.add_argument( - "--quick-inventory", + "--inventory", "-i", action="store_true", - help="Run Prowler Quick Inventory. The inventory will be stored in an output csv by default", + help="Run Prowler Inventory. The inventory will be stored in an output json file.", ) # AWS Outputs aws_outputs_subparser = aws_parser.add_argument_group("AWS Outputs to S3") diff --git a/prowler/providers/aws/lib/service/service.py b/prowler/providers/aws/lib/service/service.py index 2aaca3e394..46784f62cb 100644 --- a/prowler/providers/aws/lib/service/service.py +++ b/prowler/providers/aws/lib/service/service.py @@ -1,4 +1,7 @@ +from collections import deque from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from typing import Any, Dict from prowler.lib.logger import logger from prowler.providers.aws.aws_provider import AwsProvider @@ -101,3 +104,47 @@ class AWSService: except Exception: # Handle exceptions if necessary pass # Replace 'pass' with any additional exception handling logic. Currently handled within the called function + + def __to_dict__(self, seen=None) -> Dict[str, Any]: + if seen is None: + seen = set() + + def convert_value(value): + if isinstance(value, (AwsProvider,)): + return {} + if isinstance(value, datetime): + return value.isoformat() # Convert datetime to ISO 8601 string + elif isinstance(value, deque): + return [convert_value(item) for item in value] + elif isinstance(value, list): + return [convert_value(item) for item in value] + elif isinstance(value, tuple): + return tuple(convert_value(item) for item in value) + elif isinstance(value, dict): + # Ensure keys are strings and values are processed + return { + convert_value(str(k)): convert_value(v) for k, v in value.items() + } + elif hasattr(value, "__dict__"): + obj_id = id(value) + if obj_id in seen: + return None # Avoid infinite recursion + seen.add(obj_id) + return {key: convert_value(val) for key, val in value.__dict__.items()} + else: + return value # Handle basic types and non-serializable objects + + return { + key: convert_value(value) + for key, value in self.__dict__.items() + if key + not in [ + "audit_config", + "provider", + "session", + "regional_clients", + "client", + "thread_pool", + "fixer_config", + ] + } diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/inventory.py new file mode 100644 index 0000000000..fac27319cd --- /dev/null +++ b/prowler/providers/common/inventory.py @@ -0,0 +1,118 @@ +import importlib +import json +import os +import shutil +from collections import deque +from datetime import datetime + +from pydantic import BaseModel +from tqdm import tqdm + + +def run_prowler_inventory(checks_to_execute, provider): + output_folder_path = f"./output/inventory/{provider}" + meta_json_file = {} + + os.makedirs(output_folder_path, exist_ok=True) + + # Recursive function to handle serialization + def class_to_dict(obj, seen=None): + if seen is None: + seen = set() + + if isinstance(obj, dict): + new_dict = {} + for key, value in obj.items(): + if isinstance(key, tuple): + key = str(key) # Convert tuple to string + new_dict[key] = class_to_dict(value) + return new_dict + if isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, deque): + return list(class_to_dict(item, seen) for item in obj) + elif isinstance(obj, BaseModel): + return obj.dict() + elif isinstance(obj, (list, tuple)): + return [class_to_dict(item, seen) for item in obj] + elif hasattr(obj, "__dict__") and id(obj) not in seen: + seen.add(id(obj)) + return { + key: class_to_dict(value, seen) for key, value in obj.__dict__.items() + } + else: + return obj + + service_set = set() + + for check_name in tqdm(checks_to_execute): + try: + service = check_name.split("_")[0] + + if service in service_set: + continue + + service_set.add(service) + + service_path = f"./prowler/providers/{provider}/services/{service}" + + # List to store all _client filenames + client_files = [] + + # Walk through the directory and find all files + for root, dirs, files in os.walk(service_path): + for file in files: + if file.endswith("_client.py"): + # Append only the filename to the list (not the full path) + client_files.append(file) + + service_output_folder = f"{output_folder_path}/{service}" + + os.makedirs(service_output_folder, exist_ok=True) + + for service_client in client_files: + + service_client = service_client.split(".py")[0] + check_module_path = ( + f"prowler.providers.{provider}.services.{service}.{service_client}" + ) + + try: + lib = importlib.import_module(f"{check_module_path}") + except ModuleNotFoundError: + print(f"Module not found: {check_module_path}") + break + except Exception as e: + print(f"Error while importing module {check_module_path}: {e}") + break + + client_path = getattr(lib, f"{service_client}") + + if not meta_json_file.get(f"{service}"): + meta_json_file[f"{service}"] = [] + + # Convert to JSON + output_file = service_client.split("_client")[0] + + meta_json_file[f"{service}"].append( + f"./{service}/{output_file}_output.json" + ) + + with open( + f"{service_output_folder}/{output_file}_output.json", "w+" + ) as fp: + output = client_path.__to_dict__() + json.dump(output, fp=fp, default=str, indent=4) + + except Exception as e: + print("Exception: ", e) + + with open(f"{output_folder_path}/output_metadata.json", "w+") as fp: + json.dump(meta_json_file, fp=fp, default=str, indent=4) + + # end of all things + folder_to_compress = f"{output_folder_path}" + output_zip_file = f"{output_folder_path}/rmfx-scan-compressed" # The output file (without extension) + + # Compress the folder into a zip file + shutil.make_archive(f"{output_zip_file}", "zip", folder_to_compress) diff --git a/prowler/providers/common/quick_inventory.py b/prowler/providers/common/quick_inventory.py deleted file mode 100644 index 1023656caa..0000000000 --- a/prowler/providers/common/quick_inventory.py +++ /dev/null @@ -1,26 +0,0 @@ -import importlib -import sys - -from prowler.lib.logger import logger -from prowler.providers.aws.lib.quick_inventory.quick_inventory import quick_inventory - - -def run_provider_quick_inventory(provider, args): - """ - run_provider_quick_inventory executes the quick inventory for the provider - """ - try: - # Dynamically get the Provider quick inventory handler - provider_quick_inventory_function = f"{provider.type}_quick_inventory" - getattr(importlib.import_module(__name__), provider_quick_inventory_function)( - provider, args - ) - except Exception as error: - logger.critical( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - sys.exit(1) - - -def aws_quick_inventory(provider, args): - quick_inventory(provider, args) diff --git a/pyproject.toml b/pyproject.toml index 74a0a204d1..59c3195c0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ schema = "0.7.7" shodan = "1.31.0" slack-sdk = "3.33.1" tabulate = "0.9.0" +tqdm = "^4.66.5" tzlocal = "5.2" [tool.poetry.group.dev.dependencies] diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index f1d75b2374..f51be0c997 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -974,7 +974,7 @@ class Test_Parser: assert parsed.quick_inventory def test_aws_parser_quick_inventory_long(self): - argument = "--quick-inventory" + argument = "--inventory" command = [prowler_command, argument] parsed = self.parser.parse(command) assert parsed.quick_inventory