mirror of
https://github.com/prowler-cloud/prowler.git
synced 2025-12-19 05:17:47 +00:00
chore(examples): Scan AWS (#7064)
This commit is contained in:
Pepe Fagoaga
418
examples/sdk/aws_scan.ipynb
Normal file
418
examples/sdk/aws_scan.ipynb
Normal file
@@ -0,0 +1,418 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup and Configure Logger\n",
|
||||
"This section configures the Python logging system to filter Prowler's output messages during security scans. We set the logging level to `CRITICAL`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import logging\n",
|
||||
"\n",
|
||||
"logging.basicConfig(level=logging.CRITICAL)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Initialize AWS Provider"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import the Prowler's provider you want to scan\n",
|
||||
"from prowler.providers.aws.aws_provider import AwsProvider\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"# Path to credentials file\n",
|
||||
"credentials_path = \"./secrets-sdk/credentials.json\"\n",
|
||||
"\n",
|
||||
"# Load credentials from JSON file\n",
|
||||
"try:\n",
|
||||
" with open(credentials_path, \"r\") as f:\n",
|
||||
" aws_credentials = json.load(f)\n",
|
||||
" print(\"AWS credentials loaded successfully from file\")\n",
|
||||
"except (FileNotFoundError, json.JSONDecodeError):\n",
|
||||
" print(\"Invalid or missing JSON credentials file\")\n",
|
||||
" aws_credentials = {\n",
|
||||
" \"aws_access_key_id\": \"YOUR_ACCESS_KEY\",\n",
|
||||
" \"aws_secret_access_key\": \"YOUR_SECRET_KEY\",\n",
|
||||
" \"aws_session_token\": \"YOUR_SESSION_TOKEN\"\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"# Optional: Test the AWS provider credentials before instantiation to verify that credentials work\n",
|
||||
"aws_connection = AwsProvider.test_connection(**aws_credentials)\n",
|
||||
"print(f\"AWS Test Connection:\\n\\t- Connected: {aws_connection.is_connected}\\n\\t- Error (if any): {aws_connection.error}\\n\")\n",
|
||||
"\n",
|
||||
"# Initialize the AWS provider with static credentials\n",
|
||||
"aws = AwsProvider(**aws_credentials)\n",
|
||||
"\n",
|
||||
"# AWS Identity Information\n",
|
||||
"print(f\"AWS Identity Information:\\n\\t- Account Number: {aws.identity.account}\\n\\t- User ID: {aws.identity.user_id}\\n\")\n",
|
||||
"\n",
|
||||
"# Alternative Providers (commented out)\n",
|
||||
"# from prowler.providers.gcp.gcp_provider import GcpProvider\n",
|
||||
"# from prowler.providers.azure.azure_provider import AzureProvider\n",
|
||||
"# from prowler.providers.kubernetes.kubernetes_provider import KubernetesProvider\n",
|
||||
"# from prowler.providers.microsoft365.microsoft365_provider import Microsoft365Provider"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pprint\n",
|
||||
"pprint.pp(aws.identity)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Mutelist"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Mutelist\n",
|
||||
"from prowler.providers.aws.lib.mutelist.mutelist import AWSMutelist\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"mutelist_content = {\n",
|
||||
" \"Accounts\": {\n",
|
||||
" \"*\": {\n",
|
||||
" \"Checks\": {\n",
|
||||
" \"s3_account_level_public_access_blocks\": {\n",
|
||||
" \"Tags\": [\"*\"],\n",
|
||||
" \"Regions\": [\"*\"],\n",
|
||||
" \"Resources\": [\"*\"],\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"mutelist_object = AWSMutelist(\n",
|
||||
" mutelist_content=mutelist_content,\n",
|
||||
")\n",
|
||||
"aws._mutelist = mutelist_object"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## List Available Security Checks\n",
|
||||
"Explore different ways to list security checks by provider, service, severity, and category."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import the CheckMetadata class to list security checks\n",
|
||||
"from prowler.lib.check.models import CheckMetadata\n",
|
||||
"\n",
|
||||
"# List all available checks\n",
|
||||
"checks = CheckMetadata.list()\n",
|
||||
"print(f\"# Checks: {len(checks)}\")\n",
|
||||
"\n",
|
||||
"# List all AWS S3 checks\n",
|
||||
"aws_s3_checks = CheckMetadata.list(provider=\"aws\", service=\"s3\")\n",
|
||||
"print(f\"AWS S3 Checks:\\n\\t- {'\\n\\t- '.join(aws_s3_checks)}\")\n",
|
||||
"\n",
|
||||
"# List all critical severity checks\n",
|
||||
"critical_checks = CheckMetadata.list(provider=\"aws\", severity=\"critical\")\n",
|
||||
"print(f\"\\n# Critical Checks: {len(critical_checks)}\")\n",
|
||||
"\n",
|
||||
"# List all checks in the internet-exposed category\n",
|
||||
"internet_exposed_checks = CheckMetadata.list(provider=\"aws\", category=\"internet-exposed\")\n",
|
||||
"print(f\"\\n# Internet-Exposed Category Checks: {len(internet_exposed_checks)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Execute Security Scans\n",
|
||||
"Set up and execute security scans on AWS resources with different filtering options."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import necessary libraries for scanning\n",
|
||||
"from prowler.lib.scan.scan import Scan\n",
|
||||
"# Auxiliary libraries\n",
|
||||
"import itertools\n",
|
||||
"\n",
|
||||
"# Set up the Scan class to scan all checks for the provider\n",
|
||||
"scan = Scan(provider=aws)\n",
|
||||
"\n",
|
||||
"# Parametrize the Scan to execute several checks, services, categories, compliances, etc.\n",
|
||||
"scan_s3 = Scan(provider=aws, services=[\"s3\"], severities=[\"critical\", \"high\"])\n",
|
||||
"# scan_critical = Scan(provider=aws, severities=[\"critical\"])\n",
|
||||
"# scan_internet_exposed = Scan(provider=aws, categories=[\"internet-exposed\"])\n",
|
||||
"\n",
|
||||
"# Start the scan with the `scan` method. This returns a generator with findings and progress.\n",
|
||||
"print(\"\\n##### Scanning AWS #####\")\n",
|
||||
"all_findings = []\n",
|
||||
"total_findings = 0\n",
|
||||
"for progress, findings in scan_s3.scan():\n",
|
||||
" all_findings.extend(findings)\n",
|
||||
" total_findings += len(findings)\n",
|
||||
" print(f\"- Scan Progress: {progress}\\n- # Findings: {total_findings}\\n\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Process and Display Findings\n",
|
||||
"Process the scan results and display detailed information about each finding."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"Finding's Details:\")\n",
|
||||
"for finding in all_findings:\n",
|
||||
" print(f\"\"\"\n",
|
||||
" - Check ID: {finding.metadata.CheckID}\n",
|
||||
" - Status: {str(finding.status)}\n",
|
||||
" - Status Extended: {finding.status_extended}\n",
|
||||
" - Resource ID: {finding.resource_uid}\n",
|
||||
" - Resource Metadata: {finding.resource_metadata}\n",
|
||||
" \"\"\")\n",
|
||||
"\n",
|
||||
"# Retrieve all findings in one line\n",
|
||||
"print(\"\\n##### Getting all findings in one line #####\")\n",
|
||||
"scan_s3 = Scan(provider=aws, services=[\"s3\"], severities=[\"critical\"])\n",
|
||||
"all_findings_one_line = list(itertools.chain.from_iterable(findings for _, findings in scan_s3.scan()))\n",
|
||||
"print(f\"Total findings collected in one line: {len(all_findings_one_line)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Check Metatada\n",
|
||||
"```plain\n",
|
||||
"CheckMetadata(\n",
|
||||
" Provider='aws'\n",
|
||||
" CheckID='s3_bucket_policy_public_write_access'\n",
|
||||
" CheckTitle='Check if S3 buckets have policies which allow WRITE access.'\n",
|
||||
" CheckType=['IAM']\n",
|
||||
" CheckAliases=[]\n",
|
||||
" ServiceName='s3'\n",
|
||||
" SubServiceName=''\n",
|
||||
" ResourceIdTemplate='arn:partition:s3:::bucket_name'\n",
|
||||
" Severity=<Severity.critical: 'critical'>\n",
|
||||
" ResourceType='AwsS3Bucket'\n",
|
||||
" Description='Check if S3 buckets have policies which allow WRITE access.'\n",
|
||||
" Risk='Non intended users can put objects in a given bucket.'\n",
|
||||
" RelatedUrl=''\n",
|
||||
" Remediation=\n",
|
||||
" Remediation(\n",
|
||||
" Code=Code(\n",
|
||||
" NativeIaC=''\n",
|
||||
" Terraform=''\n",
|
||||
" CLI=''\n",
|
||||
" Other='https://docs.prowler.com/checks/aws/s3-policies/s3_18-write-permissions-public#aws-console')\n",
|
||||
" Recommendation=Recommendation(Text='Ensure proper bucket policy is in place with the least privilege principle applied.'\n",
|
||||
" Url='https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html'\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" Categories=['internet-exposed']\n",
|
||||
" DependsOn=[]\n",
|
||||
" RelatedTo=[]\n",
|
||||
" Notes=''\n",
|
||||
" # Compliance framework: A list of requirement IDs where the check is present.\n",
|
||||
" Compliance={\n",
|
||||
" \"CIS-1.10\": [\"5.2.13\"],\n",
|
||||
" \"CIS-1.8\": [\"5.2.13\"]\n",
|
||||
" }\n",
|
||||
")\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Output Formats"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import necessary libraries for output\n",
|
||||
"from prowler.lib.outputs.csv.csv import CSV\n",
|
||||
"from prowler.lib.outputs.ocsf.ocsf import OCSF\n",
|
||||
"from prowler.lib.outputs.asff.asff import ASFF # Only for AWS\n",
|
||||
"from prowler.lib.outputs.html.html import HTML\n",
|
||||
"from prowler.lib.outputs.outputs import extract_findings_statistics\n",
|
||||
"import datetime\n",
|
||||
"\n",
|
||||
"# Get current date and time in YYYY-MM-DD_HH-MM-SS format for filenames\n",
|
||||
"current_datetime = datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
|
||||
"\n",
|
||||
"# Write findings to CSV file\n",
|
||||
"print(\"Writing findings to CSV file...\")\n",
|
||||
"csv_filename = f\"./output/findings_{current_datetime}.csv\"\n",
|
||||
"csv = CSV(findings=all_findings, create_file_descriptor=True, file_path=csv_filename)\n",
|
||||
"csv.batch_write_data_to_file()\n",
|
||||
"print(f\"Done! CSV File Path: {csv._file_descriptor.name}\")\n",
|
||||
"\n",
|
||||
"# Write findings to OCSF file\n",
|
||||
"print(\"Writing findings to OCSF file...\")\n",
|
||||
"ocsf_filename = f\"./output/findings_{current_datetime}.ocsf\"\n",
|
||||
"ocsf = OCSF(findings=all_findings, create_file_descriptor=True, file_path=ocsf_filename)\n",
|
||||
"ocsf.batch_write_data_to_file()\n",
|
||||
"print(f\"Done! OCSF File Path: {ocsf._file_descriptor.name}\")\n",
|
||||
"\n",
|
||||
"# Write findings to ASFF file\n",
|
||||
"print(\"Writing findings to ASFF file...\")\n",
|
||||
"asff_filename = f\"./output/findings_{current_datetime}.asff\"\n",
|
||||
"asff = ASFF(findings=all_findings, create_file_descriptor=True, file_path=asff_filename)\n",
|
||||
"asff.batch_write_data_to_file()\n",
|
||||
"print(f\"Done! ASFF File Path: {asff._file_descriptor.name}\")\n",
|
||||
"\n",
|
||||
"# Write findings to HTML file\n",
|
||||
"print(\"Writing findings to HTML file...\")\n",
|
||||
"html_filename = f\"./output/findings_{current_datetime}.html\"\n",
|
||||
"stats = extract_findings_statistics(all_findings)\n",
|
||||
"html = HTML(findings=all_findings, create_file_descriptor=True, file_path=html_filename)\n",
|
||||
"html.batch_write_data_to_file(provider=aws, stats=stats)\n",
|
||||
"print(f\"Done! HTML File Path: {html._file_descriptor.name}\")\n",
|
||||
"\n",
|
||||
"# IMPORTANT: The create_file_descriptor parameter will be removed in 5.4.0\n",
|
||||
"# The file descriptor will be created by default when the Output class is instantiated"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Integrate with AWS S3\n",
|
||||
"Send findings to AWS S3."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import the S3 class to send findings to AWS S3\n",
|
||||
"from prowler.providers.aws.lib.s3.s3 import S3\n",
|
||||
"\n",
|
||||
"print(\"\\n##### Sending findings to S3 bucket #####\")\n",
|
||||
"generated_outputs = {\"regular\": [csv, ocsf, asff, html], \"compliance\": []}\n",
|
||||
"s3_integration = S3(aws.session.current_session, bucket_name=\"sdk-core\", output_directory=\"output\")\n",
|
||||
"s3_integration.send_to_bucket(generated_outputs)\n",
|
||||
"\n",
|
||||
"# This upload the output files to the S3 bucket. In this case:\n",
|
||||
"# sdk-core/output/csv/findings_2025-02-26_16-25-30.csv\n",
|
||||
"# sdk-core/output/ocsf/findings_2025-02-26_16-25-30.ocsf\n",
|
||||
"# sdk-core/output/asff/findings_2025-02-26_16-25-30.asff"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Integrate with AWS Security Hub\n",
|
||||
"Send findings to AWS Security Hub for centralized security monitoring."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import the SecurityHub class to send findings to AWS Security Hub\n",
|
||||
"from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub\n",
|
||||
"\n",
|
||||
"# Print message indicating the start of the process\n",
|
||||
"print(\"\\n##### Sending findings to AWS Security Hub #####\")\n",
|
||||
"\n",
|
||||
"# Get available AWS regions for Security Hub.\n",
|
||||
"# Each finding can only be sent to Security Hub within its own region.\n",
|
||||
"# Additionally, it verifies that Prowler’s integration is active in\n",
|
||||
"# Security Hub before sending\n",
|
||||
"available_regions = aws.get_available_aws_service_regions(\n",
|
||||
" \"securityhub\",\n",
|
||||
" aws.identity.partition,\n",
|
||||
" aws.identity.audited_regions,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Initialize the SecurityHub class with necessary parameters\n",
|
||||
"security_hub = SecurityHub(\n",
|
||||
" aws_account_id=aws.identity.account,\n",
|
||||
" aws_partition=aws.identity.partition,\n",
|
||||
" aws_session=aws.session.current_session,\n",
|
||||
" findings=asff.data,\n",
|
||||
" send_only_fails=False,\n",
|
||||
" aws_security_hub_available_regions=available_regions,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Send findings to AWS Security Hub\n",
|
||||
"findings_sent_to_security_hub = security_hub.batch_send_to_security_hub()\n",
|
||||
"\n",
|
||||
"# Print the number of findings sent to AWS Security Hub\n",
|
||||
"print(f\"{findings_sent_to_security_hub} findings sent to AWS Security Hub!\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "prowler-HDV3a8VZ-py3.12",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user