chore: script to generate AWS accounts list from AWS Org for bulk provisioning (#8903)

2026-07-04 19:21:51 +00:00 · 2025-10-22 22:23:14 +02:00
parent 4b160257b9
commit 8d4fa46038
9 changed files with 1029 additions and 65 deletions
@@ -19,6 +19,7 @@ A Python script to bulk-provision cloud providers in Prowler Cloud/App via REST
 - **Flexible Authentication:** Supports various authentication methods per provider
 - **Error Handling:** Comprehensive error reporting and validation
 - **Connection Testing:** Built-in provider connection verification
+- **AWS Organizations Support:** Automated YAML generation for all accounts in an AWS Organization

 ## How It Works

@@ -48,32 +49,105 @@ This two-step approach follows the Prowler API design where providers and their
   pip install -r requirements.txt
   ```

-3. Get your Prowler API token:
-   - **Prowler Cloud:** Generate token at https://api.prowler.com
-   - **Self-hosted Prowler App:** Generate token in your local instance
+3. Get your Prowler API key:
+   - **Prowler Cloud:** Create an API key at https://api.prowler.com
+   - **Self-hosted Prowler App:** Create an API key in your local instance
+   - Click **Profile** → **Account** → **Create API Key**

  ```bash
-  export PROWLER_API_TOKEN=$(curl --location 'https://api.prowler.com/api/v1/tokens' \
-    --header 'Content-Type: application/vnd.api+json' \
-    --header 'Accept: application/vnd.api+json' \
-    --data-raw '{
-      "data": {
-        "type": "tokens",
-        "attributes": {
-          "email": "your@email.com",
-          "password": "your-password"
-        }
-      }
-    }' | jq -r .data.attributes.access)
+  export PROWLER_API_KEY="pk_example-api-key"
  ```

+  For detailed instructions on creating API keys, see: https://docs.prowler.com/user-guide/providers/prowler-app-api-keys
+
+
+## AWS Organizations Integration
+
+For organizations with many AWS accounts, use the included `aws_org_generator.py` script to automatically generate configuration for all accounts in your AWS Organization.
+
+**📖 Full Guide:** See [AWS Organizations Bulk Provisioning Tutorial](https://docs.prowler.com/user-guide/tutorials/aws-organizations-bulk-provisioning) for complete documentation, examples, and troubleshooting.
+
+### Prerequisites
+
+Before using the AWS Organizations generator, deploy the ProwlerRole across all accounts using CloudFormation StackSets:
+
+**Documentation:** [Deploying Prowler IAM Roles Across AWS Organizations](https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/aws/organizations/#deploying-prowler-iam-roles-across-aws-organizations)
+
+### Quick Start
+
+1. Install additional dependencies:
+   ```bash
+   pip install -r requirements-aws-org.txt
+   ```
+
+2. Generate YAML configuration for all organization accounts:
+   ```bash
+   python aws_org_generator.py -o aws-accounts.yaml --external-id example-external-id
+   ```
+
+3. Run bulk provisioning:
+   ```bash
+   python prowler_bulk_provisioning.py aws-accounts.yaml
+   ```
+
+### AWS Organizations Generator Options
+
+```bash
+python aws_org_generator.py -o aws-accounts.yaml \
+  --role-name ProwlerRole \
+  --external-id my-external-id \
+  --exclude 123456789012 \
+  --profile org-management
+```
+
+| Option | Description | Default |
+|--------|-------------|---------|
+| `-o, --output` | Output YAML file path | `aws-org-accounts.yaml` |
+| `--role-name` | IAM role name across accounts | `ProwlerRole` |
+| `--external-id` | External ID for role assumption | None (recommended) |
+| `--session-name` | Session name for role assumption | None |
+| `--duration-seconds` | Session duration in seconds | None |
+| `--alias-format` | Alias template: `{name}`, `{id}`, `{email}` | `{name}` |
+| `--exclude` | Comma-separated account IDs to exclude | None |
+| `--include` | Comma-separated account IDs to include | None |
+| `--profile` | AWS CLI profile name | Default credentials |
+| `--region` | AWS region | `us-east-1` |
+| `--dry-run` | Print to stdout without writing | `False` |
+
+### Examples
+
+**Generate config for all accounts with custom external ID:**
+```bash
+python aws_org_generator.py -o aws-accounts.yaml --external-id prowler-2024-abc123
+```
+
+**Exclude management account:**
+```bash
+python aws_org_generator.py -o aws-accounts.yaml \
+  --external-id prowler-ext-id \
+  --exclude 123456789012
+```
+
+**Use specific AWS profile:**
+```bash
+python aws_org_generator.py -o aws-accounts.yaml \
+  --profile org-admin \
+  --external-id prowler-ext-id
+```
+
+**Custom alias format:**
+```bash
+python aws_org_generator.py -o aws-accounts.yaml \
+  --alias-format "{name}-{id}" \
+  --external-id prowler-ext-id
+```

 ## Configuration

 ### Environment Variables

 ```bash
-export PROWLER_API_TOKEN="your-prowler-token"
+export PROWLER_API_KEY="pk_example-api-key"
 export PROWLER_API_BASE="https://api.prowler.com/api/v1"  # Optional, defaults to Prowler Cloud
 ```

@@ -168,7 +242,7 @@ python prowler_bulk_provisioning.py providers.yaml \
 |--------|-------------|---------|
 | `input_file` | YAML file with provider entries | Required |
 | `--base-url` | API base URL | `https://api.prowler.com/api/v1` |
-| `--token` | Bearer token | `PROWLER_API_TOKEN` env var |
+| `--api-key` | Prowler API key | `PROWLER_API_KEY` env var |
 | `--providers-endpoint` | Providers API endpoint | `/providers` |
 | `--concurrency` | Number of concurrent requests | `5` |
 | `--timeout` | Per-request timeout in seconds | `60` |
@@ -241,8 +315,8 @@ The Prowler API supports the following authentication methods for GCP:
    # OR inline:
    # inline_json:
    #   type: "service_account"
-    #   project_id: "your-project"
-    #   private_key_id: "key-id"
+    #   project_id: "example-project"
+    #   private_key_id: "example-key-id"
    #   private_key: "-----BEGIN PRIVATE KEY-----\n..."
    #   client_email: "service-account@project.iam.gserviceaccount.com"
    #   client_id: "1234567890"
@@ -379,10 +453,10 @@ python prowler_bulk_provisioning.py providers.yaml --dry-run

 ### Common Issues

-1. **Invalid API Token**
+1. **Invalid API Key**
   ```
   Error: 401 Unauthorized
-   Solution: Check your PROWLER_API_TOKEN or --token parameter
+   Solution: Check your PROWLER_API_KEY environment variable or --api-key parameter
   ```

 2. **Network Timeouts**
@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+
+"""
+AWS Organizations Account Generator for Prowler Bulk Provisioning
+
+Generates YAML configuration for all accounts in an AWS Organization,
+ready to be used with prowler_bulk_provisioning.py.
+
+Prerequisites:
+- ProwlerRole (or custom role) must be deployed across all accounts
+- AWS credentials with Organizations read access (typically management account)
+- See: https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/aws/organizations/#deploying-prowler-iam-roles-across-aws-organizations
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Any, Dict, List, Optional
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError, NoCredentialsError
+except ImportError:
+    sys.exit(
+        "boto3 is required. Install with: pip install boto3\n"
+        "Or install all dependencies: pip install -r requirements-aws-org.txt"
+    )
+
+try:
+    import yaml
+except ImportError:
+    sys.exit("PyYAML is required. Install with: pip install pyyaml")
+
+
+def get_org_accounts(
+    profile: Optional[str] = None, region: Optional[str] = None
+) -> List[Dict[str, Any]]:
+    """
+    Retrieve all accounts from AWS Organizations.
+
+    Args:
+        profile: AWS CLI profile name
+        region: AWS region (defaults to us-east-1 for Organizations)
+
+    Returns:
+        List of account dictionaries with id, name, email, and status
+    """
+    try:
+        session = boto3.Session(profile_name=profile, region_name=region or "us-east-1")
+        client = session.client("organizations")
+
+        accounts = []
+        paginator = client.get_paginator("list_accounts")
+
+        for page in paginator.paginate():
+            for account in page["Accounts"]:
+                # Only include ACTIVE accounts
+                if account["Status"] == "ACTIVE":
+                    accounts.append(
+                        {
+                            "id": account["Id"],
+                            "name": account["Name"],
+                            "email": account["Email"],
+                            "status": account["Status"],
+                        }
+                    )
+
+        return accounts
+
+    except NoCredentialsError:
+        sys.exit(
+            "No AWS credentials found. Configure credentials using:\n"
+            "  - AWS CLI: aws configure\n"
+            "  - Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY\n"
+            "  - IAM role if running on EC2/ECS/Lambda"
+        )
+    except ClientError as e:
+        error_code = e.response["Error"]["Code"]
+        if error_code == "AccessDeniedException":
+            sys.exit(
+                "Access denied to AWS Organizations API.\n"
+                "Ensure you are using credentials from the management account\n"
+                "with permissions to call organizations:ListAccounts"
+            )
+        elif error_code == "AWSOrganizationsNotInUseException":
+            sys.exit(
+                "AWS Organizations is not enabled for this account.\n"
+                "This script requires an AWS Organization to be set up."
+            )
+        else:
+            sys.exit(f"AWS API error: {e}")
+    except Exception as e:
+        sys.exit(f"Unexpected error listing accounts: {e}")
+
+
+def generate_yaml_config(
+    accounts: List[Dict[str, Any]],
+    role_name: str = "ProwlerRole",
+    external_id: Optional[str] = None,
+    session_name: Optional[str] = None,
+    duration_seconds: Optional[int] = None,
+    alias_format: str = "{name}",
+    exclude_accounts: Optional[List[str]] = None,
+    include_accounts: Optional[List[str]] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Generate YAML configuration for Prowler bulk provisioning.
+
+    Args:
+        accounts: List of account dictionaries from get_org_accounts
+        role_name: IAM role name (default: ProwlerRole)
+        external_id: External ID for role assumption (optional but recommended)
+        session_name: Session name for role assumption (optional)
+        duration_seconds: Session duration in seconds (optional)
+        alias_format: Format string for alias (supports {name}, {id}, {email})
+        exclude_accounts: List of account IDs to exclude
+        include_accounts: List of account IDs to include (if set, only these are included)
+
+    Returns:
+        List of provider configurations ready for YAML export
+    """
+    exclude_accounts = exclude_accounts or []
+    include_accounts = include_accounts or []
+
+    providers = []
+
+    for account in accounts:
+        account_id = account["id"]
+
+        # Apply filters
+        if include_accounts and account_id not in include_accounts:
+            continue
+        if account_id in exclude_accounts:
+            continue
+
+        # Format alias using template
+        alias = alias_format.format(
+            name=account["name"], id=account_id, email=account["email"]
+        )
+
+        # Build role ARN
+        role_arn = f"arn:aws:iam::{account_id}:role/{role_name}"
+
+        # Build credentials section
+        credentials: Dict[str, Any] = {"role_arn": role_arn}
+
+        if external_id:
+            credentials["external_id"] = external_id
+
+        if session_name:
+            credentials["session_name"] = session_name
+
+        if duration_seconds:
+            credentials["duration_seconds"] = duration_seconds
+
+        # Build provider entry
+        provider = {
+            "provider": "aws",
+            "uid": account_id,
+            "alias": alias,
+            "auth_method": "role",
+            "credentials": credentials,
+        }
+
+        providers.append(provider)
+
+    return providers
+
+
+def main():
+    """Main function to generate AWS Organizations YAML configuration."""
+    parser = argparse.ArgumentParser(
+        description="Generate Prowler bulk provisioning YAML from AWS Organizations",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic usage - generate YAML for all accounts
+  python aws_org_generator.py -o aws-accounts.yaml
+
+  # Use custom role name and external ID
+  python aws_org_generator.py -o aws-accounts.yaml \\
+    --role-name ProwlerExecutionRole \\
+    --external-id my-external-id-12345
+
+  # Use specific AWS profile
+  python aws_org_generator.py -o aws-accounts.yaml \\
+    --profile org-management
+
+  # Exclude specific accounts (e.g., management account)
+  python aws_org_generator.py -o aws-accounts.yaml \\
+    --exclude 123456789012,210987654321
+
+  # Include only specific accounts
+  python aws_org_generator.py -o aws-accounts.yaml \\
+    --include 111111111111,222222222222
+
+  # Custom alias format
+  python aws_org_generator.py -o aws-accounts.yaml \\
+    --alias-format "{name}-{id}"
+
+Prerequisites:
+  1. Deploy ProwlerRole across all accounts using CloudFormation StackSets:
+     https://docs.prowler.com/projects/prowler-open-source/en/latest/tutorials/aws/organizations/#deploying-prowler-iam-roles-across-aws-organizations
+
+  2. Ensure AWS credentials have Organizations read access:
+     - organizations:ListAccounts
+     - organizations:DescribeOrganization (optional)
+        """,
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        default="aws-org-accounts.yaml",
+        help="Output YAML file path (default: aws-org-accounts.yaml)",
+    )
+
+    parser.add_argument(
+        "--role-name",
+        default="ProwlerRole",
+        help="IAM role name deployed across accounts (default: ProwlerRole)",
+    )
+
+    parser.add_argument(
+        "--external-id",
+        help="External ID for role assumption (recommended for security)",
+    )
+
+    parser.add_argument(
+        "--session-name", help="Session name for role assumption (optional)"
+    )
+
+    parser.add_argument(
+        "--duration-seconds",
+        type=int,
+        help="Session duration in seconds (optional, default: 3600)",
+    )
+
+    parser.add_argument(
+        "--alias-format",
+        default="{name}",
+        help="Alias format template. Available: {name}, {id}, {email} (default: {name})",
+    )
+
+    parser.add_argument(
+        "--exclude",
+        help="Comma-separated list of account IDs to exclude",
+    )
+
+    parser.add_argument(
+        "--include",
+        help="Comma-separated list of account IDs to include (if set, only these are processed)",
+    )
+
+    parser.add_argument(
+        "--profile",
+        help="AWS CLI profile name (uses default credentials if not specified)",
+    )
+
+    parser.add_argument(
+        "--region",
+        help="AWS region (default: us-east-1, Organizations is global but needs a region)",
+    )
+
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print configuration to stdout without writing file",
+    )
+
+    args = parser.parse_args()
+
+    # Parse exclude/include lists
+    exclude_accounts = (
+        [acc.strip() for acc in args.exclude.split(",")] if args.exclude else []
+    )
+    include_accounts = (
+        [acc.strip() for acc in args.include.split(",")] if args.include else []
+    )
+
+    print("Fetching accounts from AWS Organizations...")
+    if args.profile:
+        print(f"Using AWS profile: {args.profile}")
+
+    # Get accounts from Organizations
+    accounts = get_org_accounts(profile=args.profile, region=args.region)
+
+    if not accounts:
+        print("No active accounts found in organization.")
+        return
+
+    print(f"Found {len(accounts)} active accounts in organization")
+
+    # Generate YAML configuration
+    providers = generate_yaml_config(
+        accounts=accounts,
+        role_name=args.role_name,
+        external_id=args.external_id,
+        session_name=args.session_name,
+        duration_seconds=args.duration_seconds,
+        alias_format=args.alias_format,
+        exclude_accounts=exclude_accounts,
+        include_accounts=include_accounts,
+    )
+
+    if not providers:
+        print("No providers generated after applying filters.")
+        return
+
+    print(f"Generated configuration for {len(providers)} accounts")
+
+    # Output YAML
+    yaml_content = yaml.dump(
+        providers, default_flow_style=False, sort_keys=False, allow_unicode=True
+    )
+
+    if args.dry_run:
+        print("\n--- Generated YAML Configuration ---\n")
+        print(yaml_content)
+    else:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(yaml_content)
+        print(f"\nConfiguration written to: {args.output}")
+        print("\nNext steps:")
+        print(f"  1. Review the generated file: cat {args.output} | head -n 20")
+        print(
+            f"  2. Run bulk provisioning: python prowler_bulk_provisioning.py {args.output}"
+        )
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,49 @@
+# Example AWS Organizations Output
+#
+# This is an example of what aws_org_generator.py produces when run against
+# an AWS Organization. This file can be directly used with prowler_bulk_provisioning.py
+#
+# Generated with:
+#   python aws_org_generator.py -o aws-org-accounts.yaml \
+#     --role-name ProwlerRole \
+#     --external-id prowler-ext-id-12345
+
+- provider: aws
+  uid: '111111111111'
+  alias: Production-Account
+  auth_method: role
+  credentials:
+    role_arn: arn:aws:iam::111111111111:role/ProwlerRole
+    external_id: prowler-ext-id-12345
+
+- provider: aws
+  uid: '222222222222'
+  alias: Development-Account
+  auth_method: role
+  credentials:
+    role_arn: arn:aws:iam::222222222222:role/ProwlerRole
+    external_id: prowler-ext-id-12345
+
+- provider: aws
+  uid: '333333333333'
+  alias: Staging-Account
+  auth_method: role
+  credentials:
+    role_arn: arn:aws:iam::333333333333:role/ProwlerRole
+    external_id: prowler-ext-id-12345
+
+- provider: aws
+  uid: '444444444444'
+  alias: Security-Account
+  auth_method: role
+  credentials:
+    role_arn: arn:aws:iam::444444444444:role/ProwlerRole
+    external_id: prowler-ext-id-12345
+
+- provider: aws
+  uid: '555555555555'
+  alias: Logging-Account
+  auth_method: role
+  credentials:
+    role_arn: arn:aws:iam::555555555555:role/ProwlerRole
+    external_id: prowler-ext-id-12345
@@ -7,7 +7,7 @@ Use with extreme caution. There is no undo.

 Environment:
  PROWLER_API_BASE   (default: https://api.prowler.com/api/v1)
-  PROWLER_API_TOKEN  (required unless --token is provided)
+  PROWLER_API_KEY    (required unless --api-key is provided)

 Usage:
  python nuke_providers.py --confirm
@@ -39,12 +39,14 @@ import requests
 # ----------------------------- CLI / Utils --------------------------------- #


-def env_or_arg(token_arg: Optional[str]) -> str:
-    """Get API token from argument or environment variable."""
-    token = token_arg or os.getenv("PROWLER_API_TOKEN")
-    if not token:
-        sys.exit("Missing API token. Set --token or PROWLER_API_TOKEN.")
-    return token
+def env_or_arg(api_key_arg: Optional[str]) -> str:
+    """Get API key from argument or environment variable."""
+    api_key = api_key_arg or os.getenv("PROWLER_API_KEY")
+    if not api_key:
+        sys.exit(
+            "Missing API key. Set --api-key or PROWLER_API_KEY environment variable."
+        )
+    return api_key


 def normalize_base_url(url: str) -> str:
@@ -63,14 +65,14 @@ class ApiClient:
    """HTTP client for Prowler API."""

    base_url: str
-    token: str
+    api_key: str
    verify_ssl: bool = True
    timeout: int = 60

    def _headers(self) -> Dict[str, str]:
        """Generate HTTP headers for API requests."""
        return {
-            "Authorization": f"Bearer {self.token}",
+            "Authorization": f"Api-Key {self.api_key}",
            "Content-Type": "application/vnd.api+json",
            "Accept": "application/vnd.api+json",
        }
@@ -266,7 +268,9 @@ def main():
        help="API base URL (default: env PROWLER_API_BASE or Prowler Cloud SaaS)",
    )
    parser.add_argument(
-        "--token", default=None, help="Bearer token (default: PROWLER_API_TOKEN)"
+        "--api-key",
+        default=None,
+        help="Prowler API key (default: PROWLER_API_KEY env variable)",
    )
    parser.add_argument(
        "--filter-provider",
@@ -307,12 +311,12 @@ def main():

    args = parser.parse_args()

-    token = env_or_arg(args.token)
+    api_key = env_or_arg(args.api_key)
    base_url = normalize_base_url(args.base_url)

    client = ApiClient(
        base_url=base_url,
-        token=token,
+        api_key=api_key,
        verify_ssl=not args.insecure,
        timeout=args.timeout,
    )
@@ -132,12 +132,14 @@ def load_items(path: Path) -> List[Dict[str, Any]]:
    sys.exit(f"Unsupported input file type: {ext}")


-def env_or_arg(token_arg: Optional[str]) -> str:
-    """Get API token from argument or environment variable."""
-    token = token_arg or os.getenv("PROWLER_API_TOKEN")
-    if not token:
-        sys.exit("Missing API token. Set --token or PROWLER_API_TOKEN.")
-    return token
+def env_or_arg(api_key_arg: Optional[str]) -> str:
+    """Get API key from argument or environment variable."""
+    api_key = api_key_arg or os.getenv("PROWLER_API_KEY")
+    if not api_key:
+        sys.exit(
+            "Missing API key. Set --api-key or PROWLER_API_KEY environment variable."
+        )
+    return api_key


 def normalize_base_url(url: str) -> str:
@@ -395,14 +397,14 @@ class ApiClient:
    """HTTP client for Prowler API."""

    base_url: str
-    token: str
+    api_key: str
    verify_ssl: bool = True
    timeout: int = 60

    def _headers(self) -> Dict[str, str]:
        """Generate HTTP headers for API requests."""
        return {
-            "Authorization": f"Bearer {self.token}",
+            "Authorization": f"Api-Key {self.api_key}",
            "Content-Type": "application/vnd.api+json",
            "Accept": "application/vnd.api+json",
        }
@@ -564,7 +566,9 @@ def main():
        help="API base URL (default: env PROWLER_API_BASE or Prowler Cloud SaaS).",
    )
    parser.add_argument(
-        "--token", default=None, help="Bearer token (default: PROWLER_API_TOKEN)."
+        "--api-key",
+        default=None,
+        help="Prowler API key (default: PROWLER_API_KEY env variable).",
    )
    parser.add_argument(
        "--providers-endpoint",
@@ -600,7 +604,7 @@ def main():
    )
    args = parser.parse_args()

-    token = env_or_arg(args.token)
+    api_key = env_or_arg(args.api_key)
    base_url = normalize_base_url(args.base_url)

    items = load_items(Path(args.input_file))
@@ -610,7 +614,7 @@ def main():

    client = ApiClient(
        base_url=base_url,
-        token=token,
+        api_key=api_key,
        verify_ssl=not args.insecure,
        timeout=args.timeout,
    )
@@ -0,0 +1,11 @@
+# AWS Organizations Generator Dependencies
+#
+# This extends the base requirements.txt for the AWS Organizations generator
+
+# Include base dependencies
+-r requirements.txt
+
+# AWS SDK for Python
+boto3>=1.26.0
+
+# Note: PyYAML is already included in requirements.txt