ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-01-25 02:08:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(gcp): enhance GCP APIs logic (#7046)

Browse Source
This commit is contained in:
Sergio Garcia
2025-02-28 10:10:43 +01:00
committed by GitHub
parent c411466df7
commit 2f741f35a8
5 changed files with 153 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
docs/tutorials/gcp/authentication.md
View File

@@ -25,6 +25,9 @@ Prowler will follow the same credentials search as [Google authentication librar
Those credentials must be associated to a user or service account with proper permissions to do all checks. To make sure, add the `Viewer` role to the member associated with the credentials.
???+ note
Prowler will use the enabled Google Cloud APIs to get the information needed to perform the checks.
## Impersonate Service Account
If you want to impersonate a GCP service account, you can use the `--impersonate-service-account` argument:
@@ -34,23 +37,3 @@ prowler gcp --impersonate-service-account <service-account-email>
```
This argument will use the default credentials to impersonate the service account provided.
## Service APIs
Prowler will use the Google Cloud APIs to get the information needed to perform the checks. Make sure that the following APIs are enabled in the project:
- apikeys.googleapis.com
- artifactregistry.googleapis.com
- bigquery.googleapis.com
- sqladmin.googleapis.com
- storage.googleapis.com
- compute.googleapis.com
- dataproc.googleapis.com
- dns.googleapis.com
- containerregistry.googleapis.com
- container.googleapis.com
- iam.googleapis.com
- cloudkms.googleapis.com
- logging.googleapis.com
You can enable them automatically using our script [enable_apis_in_projects.sh](https://github.com/prowler-cloud/prowler/blob/master/contrib/gcp/enable_apis_in_projects.sh)

44
prowler/providers/gcp/exceptions/exceptions.py
View File

@@ -6,14 +6,6 @@ class GCPBaseException(ProwlerException):
"""Base class for GCP Errors."""
GCP_ERROR_CODES = {
(3000, "GCPCloudResourceManagerAPINotUsedError"): {
"message": "Cloud Resource Manager API not used",
"remediation": "Enable the Cloud Resource Manager API for the project.",
},
(3001, "GCPHTTPError"): {
"message": "HTTP error",
"remediation": "Check the HTTP error and ensure the request is properly formatted.",
},
(3002, "GCPNoAccesibleProjectsError"): {
"message": "No Project IDs are active or can be accessed via Google Credentials",
"remediation": "Ensure the project is active and accessible.",
@@ -22,10 +14,6 @@ class GCPBaseException(ProwlerException):
"message": "Error setting up session",
"remediation": "Check the session setup and ensure it is properly set up.",
},
(3004, "GCPGetProjectError"): {
"message": "Error getting project",
"remediation": "Check the project and ensure it is properly set up.",
},
(3005, "GCPTestConnectionError"): {
"message": "Error testing connection to GCP",
"remediation": "Check the connection and ensure it is properly set up.",
@@ -42,10 +30,6 @@ class GCPBaseException(ProwlerException):
"message": "Provider does not match with the expected project_id",
"remediation": "Check the provider and ensure it matches the expected project_id.",
},
(3009, "GCPCloudAssetAPINotUsedError"): {
"message": "Cloud Asset API not used",
"remediation": "Enable the Cloud Asset API for the project.",
},
(3010, "GCPLoadServiceAccountKeyFromDictError"): {
"message": "Error loading Service Account Private Key credentials from dictionary",
"remediation": "Check the dictionary and ensure it contains a Service Account Private Key.",
@@ -73,20 +57,6 @@ class GCPCredentialsError(GCPBaseException):
super().__init__(code, file, original_exception, message)
class GCPCloudResourceManagerAPINotUsedError(GCPBaseException):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
3000, file=file, original_exception=original_exception, message=message
)
class GCPHTTPError(GCPBaseException):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
3001, file=file, original_exception=original_exception, message=message
)
class GCPNoAccesibleProjectsError(GCPCredentialsError):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
@@ -101,13 +71,6 @@ class GCPSetUpSessionError(GCPCredentialsError):
)
class GCPGetProjectError(GCPCredentialsError):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
3004, file=file, original_exception=original_exception, message=message
)
class GCPTestConnectionError(GCPBaseException):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
@@ -136,13 +99,6 @@ class GCPInvalidProviderIdError(GCPBaseException):
)
class GCPCloudAssetAPINotUsedError(GCPBaseException):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(
3009, file=file, original_exception=original_exception, message=message
)
class GCPLoadServiceAccountKeyFromDictError(GCPCredentialsError):
def __init__(self, file=None, original_exception=None, message=None):
super().__init__(

267
prowler/providers/gcp/gcp_provider.py
View File

@@ -1,3 +1,4 @@
import json
import os
import re
import sys
@@ -20,10 +21,6 @@ from prowler.lib.utils.utils import print_boxes
from prowler.providers.common.models import Audit_Metadata, Connection
from prowler.providers.common.provider import Provider
from prowler.providers.gcp.exceptions.exceptions import (
GCPCloudAssetAPINotUsedError,
GCPCloudResourceManagerAPINotUsedError,
GCPGetProjectError,
GCPHTTPError,
GCPInvalidProviderIdError,
GCPLoadADCFromDictError,
GCPLoadServiceAccountKeyFromDictError,
@@ -113,7 +110,6 @@ class GcpProvider(Provider):
GCPNoAccesibleProjectsError if no project IDs can be accessed via Google Credentials
GCPSetUpSessionError if an error occurs during the setup session
GCPLoadADCFromDictError if an error occurs during the loading credentials from dict
GCPGetProjectError if an error occurs during the get project
Returns:
None
@@ -182,7 +178,9 @@ class GcpProvider(Provider):
self._project_ids = []
self._projects = {}
self._excluded_project_ids = []
accessible_projects = self.get_projects(self._session, organization_id)
accessible_projects = self.get_projects(
self._session, organization_id, project_ids, credentials_file
)
if not accessible_projects:
logger.critical("No Project IDs can be accessed via Google Credentials.")
raise GCPNoAccesibleProjectsError(
@@ -500,11 +498,13 @@ class GcpProvider(Provider):
if provider_id and project_id != provider_id:
# Logic to check if the provider ID matches the project ID
GcpProvider.validate_project_id(
provider_id=provider_id, credentials=session
provider_id=provider_id,
credentials=session,
)
service = discovery.build("cloudresourcemanager", "v1", credentials=session)
request = service.projects().list()
# Test the connection using the Service Usage API since it is enabled by default
client = discovery.build("serviceusage", "v1", credentials=session)
request = client.services().list(parent=f"projects/{project_id}")
request.execute()
return Connection(is_connected=True)
@@ -524,18 +524,9 @@ class GcpProvider(Provider):
raise setup_session_error
return Connection(error=setup_session_error)
except HttpError as http_error:
if "Cloud Resource Manager API has not been used" in str(http_error):
logger.critical(
"Cloud Resource Manager API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry."
)
if raise_on_exception:
raise GCPCloudResourceManagerAPINotUsedError(
file=__file__, original_exception=http_error
)
else:
logger.critical(
f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}"
)
logger.critical(
f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}"
)
if raise_on_exception:
raise http_error
return Connection(error=http_error)
@@ -581,7 +572,10 @@ class GcpProvider(Provider):
@staticmethod
def get_projects(
credentials: Credentials, organization_id: str = None
credentials: Credentials,
organization_id: str = None,
project_ids: list = None,
credentials_file: str = None,
) -> dict[str, GCPProject]:
"""
Get the projects accessible by the provided credentials. If an organization ID is provided, only the projects under that organization are returned.
@@ -589,16 +583,12 @@ class GcpProvider(Provider):
Args:
credentials: Credentials
organization_id: str
project_ids: list
credentials_file: str
Returns:
dict[str, GCPProject]
Raises:
GCPCloudResourceManagerAPINotUsedError if the Cloud Resource Manager API has not been used before or it is disabled
GCPCloudAssetAPINotUsedError if the Cloud Asset API has not been used before or it is disabled
GCPHTTPError if an error occurs during the HTTP request
GCPGetProjectError if an error occurs during the get project
Usage:
>>> GcpProvider.get_projects(credentials=credentials, organization_id=organization_id)
"""
@@ -606,112 +596,137 @@ class GcpProvider(Provider):
projects = {}
if organization_id:
# Initialize Cloud Asset Inventory API for recursive project retrieval
asset_service = discovery.build(
"cloudasset", "v1", credentials=credentials
)
# Set the scope to the specified organization and filter for projects
scope = f"organizations/{organization_id}"
request = asset_service.assets().list(
parent=scope,
assetTypes=["cloudresourcemanager.googleapis.com/Project"],
contentType="RESOURCE",
)
while request is not None:
response = request.execute()
for asset in response.get("assets", []):
# Extract labels and other project details
labels = {
k: v
for k, v in asset["resource"]["data"]
.get("labels", {})
.items()
}
project_id = asset["resource"]["data"]["projectId"]
gcp_project = GCPProject(
number=asset["resource"]["data"]["projectNumber"],
id=project_id,
name=asset["resource"]["data"].get("name", project_id),
lifecycle_state=asset["resource"]["data"].get(
"lifecycleState"
),
labels=labels,
)
gcp_project.organization = GCPOrganization(
id=organization_id, name=f"organizations/{organization_id}"
)
projects[project_id] = gcp_project
request = asset_service.assets().list_next(
previous_request=request, previous_response=response
try:
# Initialize Cloud Asset Inventory API for recursive project retrieval
asset_service = discovery.build(
"cloudasset", "v1", credentials=credentials
)
# Set the scope to the specified organization and filter for projects
scope = f"organizations/{organization_id}"
request = asset_service.assets().list(
parent=scope,
assetTypes=["cloudresourcemanager.googleapis.com/Project"],
contentType="RESOURCE",
)
else:
# Initialize Cloud Resource Manager API for simple project listing
service = discovery.build(
"cloudresourcemanager", "v1", credentials=credentials
)
request = service.projects().list()
while request is not None:
response = request.execute()
while request is not None:
response = request.execute()
for project in response.get("projects", []):
# Extract labels and other project details
labels = {k: v for k, v in project.get("labels", {}).items()}
project_id = project["projectId"]
gcp_project = GCPProject(
number=project["projectNumber"],
id=project_id,
name=project.get("name", project_id),
lifecycle_state=project["lifecycleState"],
labels=labels,
)
# Set organization if present in the project metadata
if (
"parent" in project
and project["parent"].get("type") == "organization"
):
parent_org_id = project["parent"]["id"]
for asset in response.get("assets", []):
# Extract labels and other project details
labels = {
k: v
for k, v in asset["resource"]["data"]
.get("labels", {})
.items()
}
project_id = asset["resource"]["data"]["projectId"]
gcp_project = GCPProject(
number=asset["resource"]["data"]["projectNumber"],
id=project_id,
name=asset["resource"]["data"].get("name", project_id),
lifecycle_state=asset["resource"]["data"].get(
"lifecycleState"
),
labels=labels,
)
gcp_project.organization = GCPOrganization(
id=parent_org_id, name=f"organizations/{parent_org_id}"
id=organization_id,
name=f"organizations/{organization_id}",
)
projects[project_id] = gcp_project
projects[project_id] = gcp_project
request = service.projects().list_next(
previous_request=request, previous_response=response
)
except HttpError as http_error:
if "Cloud Resource Manager API has not been used" in str(http_error):
logger.critical(
"Cloud Resource Manager API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry."
)
raise GCPCloudResourceManagerAPINotUsedError(
file=__file__, original_exception=http_error
)
elif "Cloud Asset API has not been used" in str(http_error):
logger.critical(
"Cloud Asset API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry."
)
raise GCPCloudAssetAPINotUsedError(
file=__file__, original_exception=http_error
)
request = asset_service.assets().list_next(
previous_request=request, previous_response=response
)
except HttpError as http_error:
if "Cloud Asset API has not been used" in str(http_error):
logger.error(
f"Projects cannot be retrieved from the Organization since Cloud Asset API has not been used before or it is disabled [{http_error.__traceback__.tb_lineno}]. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry."
)
else:
logger.error(
f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}"
)
else:
logger.error(
f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}"
)
raise GCPHTTPError(file=__file__, original_exception=http_error)
try:
# Initialize Cloud Resource Manager API for simple project listing
service = discovery.build(
"cloudresourcemanager", "v1", credentials=credentials
)
request = service.projects().list()
while request is not None:
response = request.execute()
for project in response.get("projects", []):
# Extract labels and other project details
labels = {
k: v for k, v in project.get("labels", {}).items()
}
project_id = project["projectId"]
gcp_project = GCPProject(
number=project["projectNumber"],
id=project_id,
name=project.get("name", project_id),
lifecycle_state=project["lifecycleState"],
labels=labels,
)
# Set organization if present in the project metadata
if (
"parent" in project
and project["parent"].get("type") == "organization"
):
parent_org_id = project["parent"]["id"]
gcp_project.organization = GCPOrganization(
id=parent_org_id,
name=f"organizations/{parent_org_id}",
)
projects[project_id] = gcp_project
request = service.projects().list_next(
previous_request=request, previous_response=response
)
except HttpError as http_error:
if "Cloud Resource Manager API has not been used" in str(
http_error
):
logger.error(
f"Project information cannot be retrieved since Cloud Resource Manager API has not been used before or it is disabled [{http_error.__traceback__.tb_lineno}]. Enable it by visiting https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/ then retry."
)
else:
logger.error(
f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}"
)
if not projects:
# If no projects were able to be accessed via API, add them manually if provided by the user in arguments
if project_ids:
for input_project in project_ids:
projects[input_project] = GCPProject(
id=input_project,
name=input_project,
number=0,
labels={},
lifecycle_state="ACTIVE",
)
# If no projects were able to be accessed via API, add them manually from the credentials file
elif credentials_file:
with open(credentials_file, "r", encoding="utf-8") as file:
project_id = json.load(file)["project_id"]
projects[project_id] = GCPProject(
id=project_id,
name=project_id,
number=0,
labels={},
lifecycle_state="ACTIVE",
)
except Exception as error:
logger.critical(
f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
)
raise GCPGetProjectError(file=__file__, original_exception=error)
finally:
return projects
@@ -722,10 +737,6 @@ class GcpProvider(Provider):
Returns:
None
Raises:
GCPHTTPError if an error occurs during the HTTP request
GCPGetProjectError if an error occurs during the get project
Usage:
>>> GcpProvider.update_projects_with_organizations()
"""
@@ -835,7 +846,9 @@ class GcpProvider(Provider):
"""
available_projects = list(
GcpProvider.get_projects(credentials=credentials).keys()
GcpProvider.get_projects(
credentials=credentials, project_ids=[provider_id]
).keys()
)
if len(available_projects) == 0:

5
prowler/providers/gcp/lib/service/service.py
View File

@@ -2,7 +2,6 @@ import threading
import google_auth_httplib2
import httplib2
from colorama import Fore, Style
from google.oauth2.credentials import Credentials
from googleapiclient import discovery
from googleapiclient.discovery import Resource
@@ -66,8 +65,8 @@ class GCPService:
if response.get("state") != "DISABLED":
project_ids.append(project_id)
else:
print(
f"\n{Fore.YELLOW}{self.service} API {Style.RESET_ALL}has not been used in project {project_id} before or it is disabled.\nEnable it by visiting https://console.developers.google.com/apis/api/{self.service}.googleapis.com/overview?project={project_id} then retry."
logger.error(
f"{self.service} API has not been used in project {project_id} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/{self.service}.googleapis.com/overview?project={project_id} then retry."
)
except Exception as error:
logger.error(

13
prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py
View File

@@ -42,11 +42,14 @@ class CloudResourceManager(GCPService):
def _get_organizations(self):
try:
response = self.client.organizations().search().execute()
for org in response.get("organizations", []):
self.organizations.append(
Organization(id=org["name"].split("/")[-1], name=org["displayName"])
)
if self.project_ids:
response = self.client.organizations().search().execute()
for org in response.get("organizations", []):
self.organizations.append(
Organization(
id=org["name"].split("/")[-1], name=org["displayName"]
)
)
except Exception as error:
logger.error(
f"{self.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"