From 115169a596a6c49269e059a6a6f32cdac158c6a7 Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Fri, 28 Feb 2025 10:10:43 +0100 Subject: [PATCH] chore(gcp): enhance GCP APIs logic (#7046) --- docs/tutorials/gcp/authentication.md | 23 +- .../providers/gcp/exceptions/exceptions.py | 44 --- prowler/providers/gcp/gcp_provider.py | 267 +++++++++--------- prowler/providers/gcp/lib/service/service.py | 5 +- .../cloudresourcemanager_service.py | 13 +- 5 files changed, 153 insertions(+), 199 deletions(-) diff --git a/docs/tutorials/gcp/authentication.md b/docs/tutorials/gcp/authentication.md index 1ba42c1020..85707a8f05 100644 --- a/docs/tutorials/gcp/authentication.md +++ b/docs/tutorials/gcp/authentication.md @@ -25,6 +25,9 @@ Prowler will follow the same credentials search as [Google authentication librar Those credentials must be associated to a user or service account with proper permissions to do all checks. To make sure, add the `Viewer` role to the member associated with the credentials. +???+ note + Prowler will use the enabled Google Cloud APIs to get the information needed to perform the checks. + ## Impersonate Service Account If you want to impersonate a GCP service account, you can use the `--impersonate-service-account` argument: @@ -34,23 +37,3 @@ prowler gcp --impersonate-service-account ``` This argument will use the default credentials to impersonate the service account provided. - -## Service APIs - -Prowler will use the Google Cloud APIs to get the information needed to perform the checks. Make sure that the following APIs are enabled in the project: - -- apikeys.googleapis.com -- artifactregistry.googleapis.com -- bigquery.googleapis.com -- sqladmin.googleapis.com -- storage.googleapis.com -- compute.googleapis.com -- dataproc.googleapis.com -- dns.googleapis.com -- containerregistry.googleapis.com -- container.googleapis.com -- iam.googleapis.com -- cloudkms.googleapis.com -- logging.googleapis.com - -You can enable them automatically using our script [enable_apis_in_projects.sh](https://github.com/prowler-cloud/prowler/blob/master/contrib/gcp/enable_apis_in_projects.sh) diff --git a/prowler/providers/gcp/exceptions/exceptions.py b/prowler/providers/gcp/exceptions/exceptions.py index 8b0c2bfdc6..5c5845951c 100644 --- a/prowler/providers/gcp/exceptions/exceptions.py +++ b/prowler/providers/gcp/exceptions/exceptions.py @@ -6,14 +6,6 @@ class GCPBaseException(ProwlerException): """Base class for GCP Errors.""" GCP_ERROR_CODES = { - (3000, "GCPCloudResourceManagerAPINotUsedError"): { - "message": "Cloud Resource Manager API not used", - "remediation": "Enable the Cloud Resource Manager API for the project.", - }, - (3001, "GCPHTTPError"): { - "message": "HTTP error", - "remediation": "Check the HTTP error and ensure the request is properly formatted.", - }, (3002, "GCPNoAccesibleProjectsError"): { "message": "No Project IDs are active or can be accessed via Google Credentials", "remediation": "Ensure the project is active and accessible.", @@ -22,10 +14,6 @@ class GCPBaseException(ProwlerException): "message": "Error setting up session", "remediation": "Check the session setup and ensure it is properly set up.", }, - (3004, "GCPGetProjectError"): { - "message": "Error getting project", - "remediation": "Check the project and ensure it is properly set up.", - }, (3005, "GCPTestConnectionError"): { "message": "Error testing connection to GCP", "remediation": "Check the connection and ensure it is properly set up.", @@ -42,10 +30,6 @@ class GCPBaseException(ProwlerException): "message": "Provider does not match with the expected project_id", "remediation": "Check the provider and ensure it matches the expected project_id.", }, - (3009, "GCPCloudAssetAPINotUsedError"): { - "message": "Cloud Asset API not used", - "remediation": "Enable the Cloud Asset API for the project.", - }, (3010, "GCPLoadServiceAccountKeyFromDictError"): { "message": "Error loading Service Account Private Key credentials from dictionary", "remediation": "Check the dictionary and ensure it contains a Service Account Private Key.", @@ -73,20 +57,6 @@ class GCPCredentialsError(GCPBaseException): super().__init__(code, file, original_exception, message) -class GCPCloudResourceManagerAPINotUsedError(GCPBaseException): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3000, file=file, original_exception=original_exception, message=message - ) - - -class GCPHTTPError(GCPBaseException): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3001, file=file, original_exception=original_exception, message=message - ) - - class GCPNoAccesibleProjectsError(GCPCredentialsError): def __init__(self, file=None, original_exception=None, message=None): super().__init__( @@ -101,13 +71,6 @@ class GCPSetUpSessionError(GCPCredentialsError): ) -class GCPGetProjectError(GCPCredentialsError): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3004, file=file, original_exception=original_exception, message=message - ) - - class GCPTestConnectionError(GCPBaseException): def __init__(self, file=None, original_exception=None, message=None): super().__init__( @@ -136,13 +99,6 @@ class GCPInvalidProviderIdError(GCPBaseException): ) -class GCPCloudAssetAPINotUsedError(GCPBaseException): - def __init__(self, file=None, original_exception=None, message=None): - super().__init__( - 3009, file=file, original_exception=original_exception, message=message - ) - - class GCPLoadServiceAccountKeyFromDictError(GCPCredentialsError): def __init__(self, file=None, original_exception=None, message=None): super().__init__( diff --git a/prowler/providers/gcp/gcp_provider.py b/prowler/providers/gcp/gcp_provider.py index 0154a39507..ee3a1ad4d3 100644 --- a/prowler/providers/gcp/gcp_provider.py +++ b/prowler/providers/gcp/gcp_provider.py @@ -1,3 +1,4 @@ +import json import os import re import sys @@ -20,10 +21,6 @@ from prowler.lib.utils.utils import print_boxes from prowler.providers.common.models import Audit_Metadata, Connection from prowler.providers.common.provider import Provider from prowler.providers.gcp.exceptions.exceptions import ( - GCPCloudAssetAPINotUsedError, - GCPCloudResourceManagerAPINotUsedError, - GCPGetProjectError, - GCPHTTPError, GCPInvalidProviderIdError, GCPLoadADCFromDictError, GCPLoadServiceAccountKeyFromDictError, @@ -113,7 +110,6 @@ class GcpProvider(Provider): GCPNoAccesibleProjectsError if no project IDs can be accessed via Google Credentials GCPSetUpSessionError if an error occurs during the setup session GCPLoadADCFromDictError if an error occurs during the loading credentials from dict - GCPGetProjectError if an error occurs during the get project Returns: None @@ -182,7 +178,9 @@ class GcpProvider(Provider): self._project_ids = [] self._projects = {} self._excluded_project_ids = [] - accessible_projects = self.get_projects(self._session, organization_id) + accessible_projects = self.get_projects( + self._session, organization_id, project_ids, credentials_file + ) if not accessible_projects: logger.critical("No Project IDs can be accessed via Google Credentials.") raise GCPNoAccesibleProjectsError( @@ -500,11 +498,13 @@ class GcpProvider(Provider): if provider_id and project_id != provider_id: # Logic to check if the provider ID matches the project ID GcpProvider.validate_project_id( - provider_id=provider_id, credentials=session + provider_id=provider_id, + credentials=session, ) - service = discovery.build("cloudresourcemanager", "v1", credentials=session) - request = service.projects().list() + # Test the connection using the Service Usage API since it is enabled by default + client = discovery.build("serviceusage", "v1", credentials=session) + request = client.services().list(parent=f"projects/{project_id}") request.execute() return Connection(is_connected=True) @@ -524,18 +524,9 @@ class GcpProvider(Provider): raise setup_session_error return Connection(error=setup_session_error) except HttpError as http_error: - if "Cloud Resource Manager API has not been used" in str(http_error): - logger.critical( - "Cloud Resource Manager API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry." - ) - if raise_on_exception: - raise GCPCloudResourceManagerAPINotUsedError( - file=__file__, original_exception=http_error - ) - else: - logger.critical( - f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" - ) + logger.critical( + f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" + ) if raise_on_exception: raise http_error return Connection(error=http_error) @@ -581,7 +572,10 @@ class GcpProvider(Provider): @staticmethod def get_projects( - credentials: Credentials, organization_id: str = None + credentials: Credentials, + organization_id: str = None, + project_ids: list = None, + credentials_file: str = None, ) -> dict[str, GCPProject]: """ Get the projects accessible by the provided credentials. If an organization ID is provided, only the projects under that organization are returned. @@ -589,16 +583,12 @@ class GcpProvider(Provider): Args: credentials: Credentials organization_id: str + project_ids: list + credentials_file: str Returns: dict[str, GCPProject] - Raises: - GCPCloudResourceManagerAPINotUsedError if the Cloud Resource Manager API has not been used before or it is disabled - GCPCloudAssetAPINotUsedError if the Cloud Asset API has not been used before or it is disabled - GCPHTTPError if an error occurs during the HTTP request - GCPGetProjectError if an error occurs during the get project - Usage: >>> GcpProvider.get_projects(credentials=credentials, organization_id=organization_id) """ @@ -606,112 +596,137 @@ class GcpProvider(Provider): projects = {} if organization_id: - # Initialize Cloud Asset Inventory API for recursive project retrieval - asset_service = discovery.build( - "cloudasset", "v1", credentials=credentials - ) - # Set the scope to the specified organization and filter for projects - scope = f"organizations/{organization_id}" - request = asset_service.assets().list( - parent=scope, - assetTypes=["cloudresourcemanager.googleapis.com/Project"], - contentType="RESOURCE", - ) - - while request is not None: - response = request.execute() - - for asset in response.get("assets", []): - # Extract labels and other project details - labels = { - k: v - for k, v in asset["resource"]["data"] - .get("labels", {}) - .items() - } - project_id = asset["resource"]["data"]["projectId"] - gcp_project = GCPProject( - number=asset["resource"]["data"]["projectNumber"], - id=project_id, - name=asset["resource"]["data"].get("name", project_id), - lifecycle_state=asset["resource"]["data"].get( - "lifecycleState" - ), - labels=labels, - ) - gcp_project.organization = GCPOrganization( - id=organization_id, name=f"organizations/{organization_id}" - ) - - projects[project_id] = gcp_project - - request = asset_service.assets().list_next( - previous_request=request, previous_response=response + try: + # Initialize Cloud Asset Inventory API for recursive project retrieval + asset_service = discovery.build( + "cloudasset", "v1", credentials=credentials + ) + # Set the scope to the specified organization and filter for projects + scope = f"organizations/{organization_id}" + request = asset_service.assets().list( + parent=scope, + assetTypes=["cloudresourcemanager.googleapis.com/Project"], + contentType="RESOURCE", ) - else: - # Initialize Cloud Resource Manager API for simple project listing - service = discovery.build( - "cloudresourcemanager", "v1", credentials=credentials - ) - request = service.projects().list() + while request is not None: + response = request.execute() - while request is not None: - response = request.execute() - - for project in response.get("projects", []): - # Extract labels and other project details - labels = {k: v for k, v in project.get("labels", {}).items()} - project_id = project["projectId"] - gcp_project = GCPProject( - number=project["projectNumber"], - id=project_id, - name=project.get("name", project_id), - lifecycle_state=project["lifecycleState"], - labels=labels, - ) - - # Set organization if present in the project metadata - if ( - "parent" in project - and project["parent"].get("type") == "organization" - ): - parent_org_id = project["parent"]["id"] + for asset in response.get("assets", []): + # Extract labels and other project details + labels = { + k: v + for k, v in asset["resource"]["data"] + .get("labels", {}) + .items() + } + project_id = asset["resource"]["data"]["projectId"] + gcp_project = GCPProject( + number=asset["resource"]["data"]["projectNumber"], + id=project_id, + name=asset["resource"]["data"].get("name", project_id), + lifecycle_state=asset["resource"]["data"].get( + "lifecycleState" + ), + labels=labels, + ) gcp_project.organization = GCPOrganization( - id=parent_org_id, name=f"organizations/{parent_org_id}" + id=organization_id, + name=f"organizations/{organization_id}", ) - projects[project_id] = gcp_project + projects[project_id] = gcp_project - request = service.projects().list_next( - previous_request=request, previous_response=response - ) - - except HttpError as http_error: - if "Cloud Resource Manager API has not been used" in str(http_error): - logger.critical( - "Cloud Resource Manager API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry." - ) - raise GCPCloudResourceManagerAPINotUsedError( - file=__file__, original_exception=http_error - ) - elif "Cloud Asset API has not been used" in str(http_error): - logger.critical( - "Cloud Asset API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry." - ) - raise GCPCloudAssetAPINotUsedError( - file=__file__, original_exception=http_error - ) + request = asset_service.assets().list_next( + previous_request=request, previous_response=response + ) + except HttpError as http_error: + if "Cloud Asset API has not been used" in str(http_error): + logger.error( + f"Projects cannot be retrieved from the Organization since Cloud Asset API has not been used before or it is disabled [{http_error.__traceback__.tb_lineno}]. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry." + ) + else: + logger.error( + f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" + ) else: - logger.error( - f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" - ) - raise GCPHTTPError(file=__file__, original_exception=http_error) + try: + # Initialize Cloud Resource Manager API for simple project listing + service = discovery.build( + "cloudresourcemanager", "v1", credentials=credentials + ) + request = service.projects().list() + + while request is not None: + response = request.execute() + + for project in response.get("projects", []): + # Extract labels and other project details + labels = { + k: v for k, v in project.get("labels", {}).items() + } + project_id = project["projectId"] + gcp_project = GCPProject( + number=project["projectNumber"], + id=project_id, + name=project.get("name", project_id), + lifecycle_state=project["lifecycleState"], + labels=labels, + ) + + # Set organization if present in the project metadata + if ( + "parent" in project + and project["parent"].get("type") == "organization" + ): + parent_org_id = project["parent"]["id"] + gcp_project.organization = GCPOrganization( + id=parent_org_id, + name=f"organizations/{parent_org_id}", + ) + + projects[project_id] = gcp_project + + request = service.projects().list_next( + previous_request=request, previous_response=response + ) + except HttpError as http_error: + if "Cloud Resource Manager API has not been used" in str( + http_error + ): + logger.error( + f"Project information cannot be retrieved since Cloud Resource Manager API has not been used before or it is disabled [{http_error.__traceback__.tb_lineno}]. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry." + ) + else: + logger.error( + f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}" + ) + if not projects: + # If no projects were able to be accessed via API, add them manually if provided by the user in arguments + if project_ids: + for input_project in project_ids: + projects[input_project] = GCPProject( + id=input_project, + name=input_project, + number=0, + labels={}, + lifecycle_state="ACTIVE", + ) + # If no projects were able to be accessed via API, add them manually from the credentials file + elif credentials_file: + with open(credentials_file, "r", encoding="utf-8") as file: + project_id = json.load(file)["project_id"] + projects[project_id] = GCPProject( + id=project_id, + name=project_id, + number=0, + labels={}, + lifecycle_state="ACTIVE", + ) except Exception as error: logger.critical( f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - raise GCPGetProjectError(file=__file__, original_exception=error) finally: return projects @@ -722,10 +737,6 @@ class GcpProvider(Provider): Returns: None - Raises: - GCPHTTPError if an error occurs during the HTTP request - GCPGetProjectError if an error occurs during the get project - Usage: >>> GcpProvider.update_projects_with_organizations() """ @@ -835,7 +846,9 @@ class GcpProvider(Provider): """ available_projects = list( - GcpProvider.get_projects(credentials=credentials).keys() + GcpProvider.get_projects( + credentials=credentials, project_ids=[provider_id] + ).keys() ) if len(available_projects) == 0: diff --git a/prowler/providers/gcp/lib/service/service.py b/prowler/providers/gcp/lib/service/service.py index d163e4781a..ce78476f37 100644 --- a/prowler/providers/gcp/lib/service/service.py +++ b/prowler/providers/gcp/lib/service/service.py @@ -2,7 +2,6 @@ import threading import google_auth_httplib2 import httplib2 -from colorama import Fore, Style from google.oauth2.credentials import Credentials from googleapiclient import discovery from googleapiclient.discovery import Resource @@ -66,8 +65,8 @@ class GCPService: if response.get("state") != "DISABLED": project_ids.append(project_id) else: - print( - f"\n{Fore.YELLOW}{self.service} API {Style.RESET_ALL}has not been used in project {project_id} before or it is disabled.\nEnable it by visiting https://console.developers.google.com/apis/api/{self.service}.googleapis.com/overview?project={project_id} then retry." + logger.error( + f"{self.service} API has not been used in project {project_id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/{self.service}.googleapis.com/overview?project={project_id} then retry." ) except Exception as error: logger.error( diff --git a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py index 2ca72dee3e..28e2f4d832 100644 --- a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py +++ b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py @@ -42,11 +42,14 @@ class CloudResourceManager(GCPService): def _get_organizations(self): try: - response = self.client.organizations().search().execute() - for org in response.get("organizations", []): - self.organizations.append( - Organization(id=org["name"].split("/")[-1], name=org["displayName"]) - ) + if self.project_ids: + response = self.client.organizations().search().execute() + for org in response.get("organizations", []): + self.organizations.append( + Organization( + id=org["name"].split("/")[-1], name=org["displayName"] + ) + ) except Exception as error: logger.error( f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"