From f01910e4f26459a62a2c7a62a16f2773f4de377f Mon Sep 17 00:00:00 2001 From: Sergio Garcia <38561120+sergargar@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:11:53 -0500 Subject: [PATCH] feat(gcp): add `--organization-id` flag (#5524) --- docs/tutorials/gcp/organization.md | 22 +++ mkdocs.yml | 1 + prowler/providers/common/provider.py | 1 + .../providers/gcp/exceptions/exceptions.py | 11 ++ prowler/providers/gcp/gcp_provider.py | 128 +++++++++++++----- .../providers/gcp/lib/arguments/arguments.py | 8 ++ tests/lib/cli/parser_test.py | 8 ++ tests/providers/gcp/gcp_provider_test.py | 86 +++++++++++- 8 files changed, 232 insertions(+), 33 deletions(-) create mode 100644 docs/tutorials/gcp/organization.md diff --git a/docs/tutorials/gcp/organization.md b/docs/tutorials/gcp/organization.md new file mode 100644 index 0000000000..43a58aa7b9 --- /dev/null +++ b/docs/tutorials/gcp/organization.md @@ -0,0 +1,22 @@ +# GCP Organization + +By default, Prowler scans all Google Cloud projects accessible to the authenticated user. + +To limit the scan to projects within a specific Google Cloud organization, use the `--organization-id` option with the GCP organization ID: + +```console +prowler gcp --organization-id organization-id +``` + +???+ warning + Make sure that the used credentials have the role Cloud Asset Viewer (`roles/cloudasset.viewer`) or Cloud Asset Owner (`roles/cloudasset.owner`) on the organization level. + +???+ note + With this option, Prowler retrieves all projects within the specified organization, including those organized in folders and nested subfolders. This ensures that every project under the organization’s hierarchy is scanned, providing full visibility across the entire organization. + +???+ note + To find the organization ID, use the following command: + + ```console + gcloud organizations list + ``` diff --git a/mkdocs.yml b/mkdocs.yml index d696191908..bc9ce5604b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - Google Cloud: - Authentication: tutorials/gcp/authentication.md - Projects: tutorials/gcp/projects.md + - Organization: tutorials/gcp/organization.md - Kubernetes: - In-Cluster Execution: tutorials/kubernetes/in-cluster.md - Non In-Cluster Execution: tutorials/kubernetes/outside-cluster.md diff --git a/prowler/providers/common/provider.py b/prowler/providers/common/provider.py index 832d6268f8..3681cdcc26 100644 --- a/prowler/providers/common/provider.py +++ b/prowler/providers/common/provider.py @@ -192,6 +192,7 @@ class Provider(ABC): ) elif "gcp" in provider_class_name.lower(): provider_class( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file, diff --git a/prowler/providers/gcp/exceptions/exceptions.py b/prowler/providers/gcp/exceptions/exceptions.py index 03a2281258..c61bc525f8 100644 --- a/prowler/providers/gcp/exceptions/exceptions.py +++ b/prowler/providers/gcp/exceptions/exceptions.py @@ -42,6 +42,10 @@ class GCPBaseException(ProwlerException): "message": "Provider does not match with the expected project_id", "remediation": "Check the provider and ensure it matches the expected project_id.", }, + (3009, "GCPCloudAssetAPINotUsedError"): { + "message": "Cloud Asset API not used", + "remediation": "Enable the Cloud Asset API for the project.", + }, } def __init__(self, code, file=None, original_exception=None, message=None): @@ -126,3 +130,10 @@ class GCPInvalidProviderIdError(GCPBaseException): super().__init__( 3008, file=file, original_exception=original_exception, message=message ) + + +class GCPCloudAssetAPINotUsedError(GCPBaseException): + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 3009, file=file, original_exception=original_exception, message=message + ) diff --git a/prowler/providers/gcp/gcp_provider.py b/prowler/providers/gcp/gcp_provider.py index 6cf80e67ef..847480bdbf 100644 --- a/prowler/providers/gcp/gcp_provider.py +++ b/prowler/providers/gcp/gcp_provider.py @@ -20,6 +20,7 @@ from prowler.lib.utils.utils import print_boxes from prowler.providers.common.models import Audit_Metadata, Connection from prowler.providers.common.provider import Provider from prowler.providers.gcp.exceptions.exceptions import ( + GCPCloudAssetAPINotUsedError, GCPCloudResourceManagerAPINotUsedError, GCPGetProjectError, GCPHTTPError, @@ -47,6 +48,7 @@ class GcpProvider(Provider): def __init__( self, + organization_id: str = None, project_ids: list = None, excluded_project_ids: list = None, credentials_file: str = None, @@ -65,6 +67,7 @@ class GcpProvider(Provider): GCP Provider constructor Args: + organization_id: str project_ids: list excluded_project_ids: list credentials_file: str @@ -95,7 +98,7 @@ class GcpProvider(Provider): self._project_ids = [] self._projects = {} self._excluded_project_ids = [] - accessible_projects = self.get_projects(self._session) + accessible_projects = self.get_projects(self._session, organization_id) if not accessible_projects: logger.critical("No Project IDs can be accessed via Google Credentials.") raise GCPNoAccesibleProjectsError( @@ -407,47 +410,101 @@ class GcpProvider(Provider): print_boxes(report_lines, report_title) @staticmethod - def get_projects(credentials) -> dict[str, GCPProject]: + def get_projects( + credentials: Credentials, organization_id: str + ) -> dict[str, GCPProject]: + """ + Get the projects accessible by the provided credentials. If an organization ID is provided, only the projects under that organization are returned. + Args: + credentials: Credentials + organization_id: str + Returns: + dict[str, GCPProject] + """ try: projects = {} - service = discovery.build( - "cloudresourcemanager", "v1", credentials=credentials - ) + if organization_id: + # Initialize Cloud Asset Inventory API for recursive project retrieval + asset_service = discovery.build( + "cloudasset", "v1", credentials=credentials + ) + # Set the scope to the specified organization and filter for projects + scope = f"organizations/{organization_id}" + request = asset_service.assets().list( + parent=scope, + assetTypes=["cloudresourcemanager.googleapis.com/Project"], + contentType="RESOURCE", + ) - request = service.projects().list() + while request is not None: + response = request.execute() - while request is not None: - response = request.execute() - - for project in response.get("projects", []): - labels = {} - for key, value in project.get("labels", {}).items(): - labels[key] = value - - project_id = project["projectId"] - gcp_project = GCPProject( - number=project["projectNumber"], - id=project_id, - name=project.get("name", project_id), - lifecycle_state=project["lifecycleState"], - labels=labels, - ) - - if ( - "parent" in project - and "type" in project["parent"] - and project["parent"]["type"] == "organization" - ): - organization_id = project["parent"]["id"] + for asset in response.get("assets", []): + # Extract labels and other project details + labels = { + k: v + for k, v in asset["resource"]["data"] + .get("labels", {}) + .items() + } + project_id = asset["resource"]["data"]["projectId"] + gcp_project = GCPProject( + number=asset["resource"]["data"]["projectNumber"], + id=project_id, + name=asset["resource"]["data"].get("name", project_id), + lifecycle_state=asset["resource"]["data"].get( + "lifecycleState" + ), + labels=labels, + ) gcp_project.organization = GCPOrganization( id=organization_id, name=f"organizations/{organization_id}" ) - projects[project_id] = gcp_project - request = service.projects().list_next( - previous_request=request, previous_response=response + projects[project_id] = gcp_project + + request = asset_service.assets().list_next( + previous_request=request, previous_response=response + ) + + else: + # Initialize Cloud Resource Manager API for simple project listing + service = discovery.build( + "cloudresourcemanager", "v1", credentials=credentials ) + request = service.projects().list() + + while request is not None: + response = request.execute() + + for project in response.get("projects", []): + # Extract labels and other project details + labels = {k: v for k, v in project.get("labels", {}).items()} + project_id = project["projectId"] + gcp_project = GCPProject( + number=project["projectNumber"], + id=project_id, + name=project.get("name", project_id), + lifecycle_state=project["lifecycleState"], + labels=labels, + ) + + # Set organization if present in the project metadata + if ( + "parent" in project + and project["parent"].get("type") == "organization" + ): + parent_org_id = project["parent"]["id"] + gcp_project.organization = GCPOrganization( + id=parent_org_id, name=f"organizations/{parent_org_id}" + ) + + projects[project_id] = gcp_project + + request = service.projects().list_next( + previous_request=request, previous_response=response + ) except HttpError as http_error: if "Cloud Resource Manager API has not been used" in str(http_error): @@ -457,6 +514,13 @@ class GcpProvider(Provider): raise GCPCloudResourceManagerAPINotUsedError( file=__file__, original_exception=http_error ) + elif "Cloud Asset API has not been used" in str(http_error): + logger.critical( + "Cloud Asset API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry." + ) + raise GCPCloudAssetAPINotUsedError( + file=__file__, original_exception=http_error + ) else: logger.error( f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" diff --git a/prowler/providers/gcp/lib/arguments/arguments.py b/prowler/providers/gcp/lib/arguments/arguments.py index e9239cb3ab..24b0958b83 100644 --- a/prowler/providers/gcp/lib/arguments/arguments.py +++ b/prowler/providers/gcp/lib/arguments/arguments.py @@ -18,6 +18,14 @@ def init_parser(self): metavar="SERVICE_ACCOUNT", help="Impersonate a Google Service Account", ) + # Organizations + gcp_organization_subparser = gcp_parser.add_argument_group("Organization") + gcp_organization_subparser.add_argument( + "--organization-id", + nargs="?", + metavar="ORGANIZATION_ID", + help="GCP Organization ID to be scanned by Prowler", + ) # Projects gcp_projects_subparser = gcp_parser.add_argument_group("Projects") gcp_projects_subparser.add_argument( diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index 177b9e36bc..ce5d770e33 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -1222,6 +1222,14 @@ class Test_Parser: assert parsed.provider == "gcp" assert parsed.credentials_file == file + def test_parser_gcp_organization_id(self): + argument = "--organization-id" + organization = "test_organization" + command = [prowler_command, "gcp", argument, organization] + parsed = self.parser.parse(command) + assert parsed.provider == "gcp" + assert parsed.organization_id == organization + def test_parser_gcp_project_id(self): argument = "--project-id" project_1 = "test_project_1" diff --git a/tests/providers/gcp/gcp_provider_test.py b/tests/providers/gcp/gcp_provider_test.py index 3290ad7d11..d32fa47b28 100644 --- a/tests/providers/gcp/gcp_provider_test.py +++ b/tests/providers/gcp/gcp_provider_test.py @@ -17,7 +17,7 @@ from prowler.providers.gcp.exceptions.exceptions import ( GCPTestConnectionError, ) from prowler.providers.gcp.gcp_provider import GcpProvider -from prowler.providers.gcp.models import GCPIdentityInfo, GCPProject +from prowler.providers.gcp.models import GCPIdentityInfo, GCPOrganization, GCPProject class TestGCPProvider: @@ -87,6 +87,7 @@ class TestGCPProvider: arguments = Namespace() arguments.project_id = [] arguments.excluded_project_id = [] + arguments.organization_id = None arguments.list_project_id = False arguments.credentials_file = "" arguments.impersonate_service_account = "" @@ -131,6 +132,7 @@ class TestGCPProvider: return_value=mocked_service, ): gcp_provider = GcpProvider( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file, @@ -168,6 +170,7 @@ class TestGCPProvider: arguments = Namespace() arguments.project_id = [] arguments.excluded_project_id = [] + arguments.organization_id = None arguments.list_project_id = False arguments.credentials_file = "test_credentials_file" arguments.impersonate_service_account = "" @@ -206,6 +209,7 @@ class TestGCPProvider: return_value=mocked_service, ): gcp_provider = GcpProvider( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file, @@ -230,6 +234,7 @@ class TestGCPProvider: arguments = Namespace() arguments.project_id = [] arguments.excluded_project_id = [] + arguments.organization_id = None arguments.list_project_id = False arguments.credentials_file = "test_credentials_file" arguments.impersonate_service_account = "test-impersonate-service-account" @@ -268,6 +273,7 @@ class TestGCPProvider: return_value=mocked_service, ): gcp_provider = GcpProvider( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file, @@ -291,6 +297,78 @@ class TestGCPProvider: == "test-impersonate-service-account" ) + def test_setup_session_with_organization_id(self): + mocked_credentials = MagicMock() + + mocked_credentials.refresh.return_value = None + mocked_credentials._service_account_email = "test-service-account-email" + + arguments = Namespace() + arguments.project_id = [] + arguments.excluded_project_id = [] + arguments.organization_id = "test-organization-id" + arguments.list_project_id = False + arguments.credentials_file = "test_credentials_file" + arguments.impersonate_service_account = "" + arguments.config_file = default_config_file_path + arguments.fixer_config = default_fixer_config_file_path + + projects = { + "test-project": GCPProject( + number="55555555", + id="project/55555555", + name="test-project", + labels={"test": "value"}, + lifecycle_state="", + organization=GCPOrganization( + id="test-organization-id", + name="test-organization", + display_name="Test Organization", + ), + ) + } + + mocked_service = MagicMock() + + mocked_service.projects.list.return_value = MagicMock( + execute=MagicMock(return_value={"projects": projects}) + ) + with patch( + "prowler.providers.gcp.gcp_provider.GcpProvider.get_projects", + return_value=projects, + ), patch( + "prowler.providers.gcp.gcp_provider.GcpProvider.update_projects_with_organizations", + return_value=None, + ), patch( + "os.path.abspath", + return_value="test_credentials_file", + ), patch( + "prowler.providers.gcp.gcp_provider.default", + return_value=(mocked_credentials, MagicMock()), + ), patch( + "prowler.providers.gcp.gcp_provider.discovery.build", + return_value=mocked_service, + ): + gcp_provider = GcpProvider( + arguments.organization_id, + arguments.project_id, + arguments.excluded_project_id, + arguments.credentials_file, + arguments.impersonate_service_account, + arguments.list_project_id, + arguments.config_file, + arguments.fixer_config, + client_id=None, + client_secret=None, + refresh_token=None, + ) + assert environ["GOOGLE_APPLICATION_CREDENTIALS"] == "test_credentials_file" + assert gcp_provider.session is not None + assert ( + gcp_provider.projects["test-project"].organization.id + == "test-organization-id" + ) + def test_print_credentials_default_options(self, capsys): mocked_credentials = MagicMock() @@ -300,6 +378,7 @@ class TestGCPProvider: arguments = Namespace() arguments.project_id = [] arguments.excluded_project_id = [] + arguments.organization_id = None arguments.list_project_id = False arguments.credentials_file = "test_credentials_file" arguments.impersonate_service_account = "" @@ -338,6 +417,7 @@ class TestGCPProvider: return_value=mocked_service, ): gcp_provider = GcpProvider( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file, @@ -369,6 +449,7 @@ class TestGCPProvider: arguments = Namespace() arguments.project_id = [] arguments.excluded_project_id = [] + arguments.organization_id = None arguments.list_project_id = False arguments.credentials_file = "test_credentials_file" arguments.impersonate_service_account = "test-impersonate-service-account" @@ -407,6 +488,7 @@ class TestGCPProvider: return_value=mocked_service, ): gcp_provider = GcpProvider( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file, @@ -438,6 +520,7 @@ class TestGCPProvider: arguments = Namespace() arguments.project_id = [] arguments.excluded_project_id = ["test-excluded-project"] + arguments.organization_id = None arguments.list_project_id = False arguments.credentials_file = "test_credentials_file" arguments.impersonate_service_account = "" @@ -484,6 +567,7 @@ class TestGCPProvider: return_value=mocked_service, ): gcp_provider = GcpProvider( + arguments.organization_id, arguments.project_id, arguments.excluded_project_id, arguments.credentials_file,