From be55fa22fd6b7743de78a965b1101d59ea8c1368 Mon Sep 17 00:00:00 2001 From: Pepe Fagoaga Date: Fri, 1 Mar 2024 13:30:09 +0100 Subject: [PATCH] chore(azure): working version executing checks (#3474) --- prowler/__main__.py | 1 - prowler/lib/check/check.py | 2 + prowler/providers/aws/aws_provider.py | 1 + prowler/providers/azure/azure_provider.py | 1 + prowler/providers/common/outputs.py | 6 +- prowler/providers/gcp/gcp_provider.py | 59 +++++++-- prowler/providers/gcp/gcp_provider_new.py | 119 ------------------ prowler/providers/gcp/lib/service/service.py | 2 +- .../gcp/services/apikeys/apikeys_service.py | 3 +- .../gcp/services/bigquery/bigquery_service.py | 3 +- .../cloudresourcemanager_service.py | 3 +- .../gcp/services/cloudsql/cloudsql_service.py | 3 +- .../cloudstorage/cloudstorage_service.py | 3 +- .../gcp/services/compute/compute_service.py | 3 +- .../gcp/services/dataproc/dataproc_service.py | 3 +- .../providers/gcp/services/dns/dns_service.py | 3 +- .../providers/gcp/services/gke/gke_service.py | 5 +- .../providers/gcp/services/iam/iam_service.py | 7 +- .../providers/gcp/services/kms/kms_service.py | 3 +- .../gcp/services/logging/logging_service.py | 3 +- .../services/monitoring/monitoring_service.py | 3 +- .../serviceusage/serviceusage_service.py | 3 +- 22 files changed, 86 insertions(+), 153 deletions(-) delete mode 100644 prowler/providers/gcp/gcp_provider_new.py diff --git a/prowler/__main__.py b/prowler/__main__.py index eaced6a1f6..b7027c9ff0 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -190,7 +190,6 @@ def prowler(): mutelist_file = global_provider.get_mutelist(args.mutelist_file) # Set output options based on the selected provider - # TODO: this is going to be removed an include in the Provider as a new common object audit_output_options = set_provider_output_options( provider, args, global_provider.identity, mutelist_file, bulk_checks_metadata diff --git a/prowler/lib/check/check.py b/prowler/lib/check/check.py index 9a48b512fd..8916f04420 100644 --- a/prowler/lib/check/check.py +++ b/prowler/lib/check/check.py @@ -522,10 +522,12 @@ def execute_checks( # If check does not exists in the provider or is from another provider except ModuleNotFoundError: + # TODO: add more loggin here, we need the original exception -- traceback.print_last() logger.error( f"Check '{check_name}' was not found for the {global_provider.provider.upper()} provider" ) except Exception as error: + # TODO: add more loggin here, we need the original exception -- traceback.print_last() logger.error( f"{check_name} - {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) diff --git a/prowler/providers/aws/aws_provider.py b/prowler/providers/aws/aws_provider.py index 1dcf1bf384..fe2f1936d6 100644 --- a/prowler/providers/aws/aws_provider.py +++ b/prowler/providers/aws/aws_provider.py @@ -40,6 +40,7 @@ from prowler.providers.aws.models import ( from prowler.providers.common.provider import Provider +# TODO: why do we have variables defined in the class not passed to the __init__??? class AwsProvider(Provider): session: AWSSession = AWSSession( session=None, session_config=None, original_session=None diff --git a/prowler/providers/azure/azure_provider.py b/prowler/providers/azure/azure_provider.py index 2a0899b911..0a1bd2ed3f 100644 --- a/prowler/providers/azure/azure_provider.py +++ b/prowler/providers/azure/azure_provider.py @@ -15,6 +15,7 @@ from prowler.providers.azure.models import AzureIdentityInfo, AzureRegionConfig from prowler.providers.common.provider import Provider +# TODO: why do we have variables defined in the class not passed to the __init__??? class AzureProvider(Provider): session: DefaultAzureCredential identity: AzureIdentityInfo diff --git a/prowler/providers/common/outputs.py b/prowler/providers/common/outputs.py index 90ba012639..463a2e309a 100644 --- a/prowler/providers/common/outputs.py +++ b/prowler/providers/common/outputs.py @@ -103,7 +103,7 @@ class Azure_Output_Options(Provider_Output_Options): class Gcp_Output_Options(Provider_Output_Options): - def __init__(self, arguments, audit_info, mutelist_file, bulk_checks_metadata): + def __init__(self, arguments, identity, mutelist_file, bulk_checks_metadata): # First call Provider_Output_Options init super().__init__(arguments, mutelist_file, bulk_checks_metadata) @@ -112,7 +112,9 @@ class Gcp_Output_Options(Provider_Output_Options): not hasattr(arguments, "output_filename") or arguments.output_filename is None ): - self.output_filename = f"prowler-output-{getattr(audit_info.credentials, '_service_account_email', 'default')}-{output_file_timestamp}" + self.output_filename = ( + f"prowler-output-{identity.profile}-{output_file_timestamp}" + ) else: self.output_filename = arguments.output_filename diff --git a/prowler/providers/gcp/gcp_provider.py b/prowler/providers/gcp/gcp_provider.py index 54a8f5f999..db359a6fbd 100644 --- a/prowler/providers/gcp/gcp_provider.py +++ b/prowler/providers/gcp/gcp_provider.py @@ -1,23 +1,41 @@ import os import sys +from dataclasses import dataclass +from typing import Any, Optional from colorama import Fore, Style from google import auth +from google.oauth2.credentials import Credentials from googleapiclient import discovery from prowler.lib.logger import logger +from prowler.providers.common.provider import Provider -class GCP_Provider: - def __init__( - self, - credentials_file: str, - input_project_ids: list, - ): +@dataclass +class GCPIdentityInfo: + profile: str + + +# TODO: why do we have variables defined in the class not passed to the __init__??? +class GcpProvider(Provider): + # TODO: should we move session and identity to the Provider parent class? + provider = "gcp" + session: Credentials + default_project_id: str + project_ids: list + # TODO: review this since we have to create an identity object + identity: GCPIdentityInfo + audit_resources: Optional[Any] + audit_metadata: Optional[Any] + audit_config: Optional[dict] + + def __init__(self, arguments): logger.info("Instantiating GCP Provider ...") - self.credentials, self.default_project_id = self.__set_credentials__( - credentials_file - ) + input_project_ids = arguments.project_ids + credentials_file = arguments.credentials_file + + self.session, self.default_project_id = self.setup_session(credentials_file) self.project_ids = [] accessible_projects = self.get_project_ids() @@ -38,7 +56,15 @@ class GCP_Provider: # If not projects were input, all accessible projects are scanned by default self.project_ids = accessible_projects - def __set_credentials__(self, credentials_file): + self.identity = GCPIdentityInfo( + profile=getattr(self.session, "_service_account_email", "default") + ) + + # TODO: move this to the parent class or the main + if not arguments.only_logs: + self.print_credentials() + + def setup_session(self, credentials_file): try: if credentials_file: self.__set_gcp_creds_env_var__(credentials_file) @@ -59,15 +85,22 @@ class GCP_Provider: client_secrets_path = os.path.abspath(credentials_file) os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = client_secrets_path - def get_credentials(self): - return self.credentials, self.default_project_id, self.project_ids + def print_credentials(self): + # Beautify audited profile, set "default" if there is no profile set + + report = f""" +This report is being generated using credentials below: + +GCP Account: {Fore.YELLOW}[{self.identity.profile}]{Style.RESET_ALL} GCP Project IDs: {Fore.YELLOW}[{", ".join(self.project_ids)}]{Style.RESET_ALL} +""" + print(report) def get_project_ids(self): try: project_ids = [] service = discovery.build( - "cloudresourcemanager", "v1", credentials=self.credentials + "cloudresourcemanager", "v1", credentials=self.session ) request = service.projects().list() diff --git a/prowler/providers/gcp/gcp_provider_new.py b/prowler/providers/gcp/gcp_provider_new.py deleted file mode 100644 index d1031b5c13..0000000000 --- a/prowler/providers/gcp/gcp_provider_new.py +++ /dev/null @@ -1,119 +0,0 @@ -import os -import sys -from typing import Any, Optional - -from colorama import Fore, Style -from google import auth -from google.oauth2.credentials import Credentials -from googleapiclient import discovery - -from prowler.lib.logger import logger -from prowler.providers.common.provider import Provider - - -class GcpProvider(Provider): - session: Credentials - default_project_id: str - project_ids: list - audit_resources: Optional[Any] - audit_metadata: Optional[Any] - audit_config: Optional[dict] - - def __init__(self, arguments): - logger.info("Instantiating GCP Provider ...") - input_project_ids = arguments.project_ids - credentials_file = arguments.credentials_file - - self.session, self.default_project_id = self.setup_session(credentials_file) - if not self.default_project_id: - logger.critical("No Project ID associated to Google Credentials.") - sys.exit(1) - - self.project_ids = [] - accessible_projects = self.get_project_ids() - if not accessible_projects: - logger.critical("No Project IDs can be accessed via Google Credentials.") - sys.exit(1) - - if input_project_ids: - for input_project in input_project_ids: - if input_project in accessible_projects: - self.project_ids.append(input_project) - else: - logger.critical( - f"Project {input_project} cannot be accessed via Google Credentials." - ) - sys.exit(1) - else: - # If not projects were input, all accessible projects are scanned by default - self.project_ids = accessible_projects - - if not arguments.only_logs: - self.print_credentials() - - def setup_session(self, credentials_file): - try: - if credentials_file: - self.__set_gcp_creds_env_var__(credentials_file) - - return auth.default( - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - except Exception as error: - logger.critical( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - sys.exit(1) - - def __set_gcp_creds_env_var__(self, credentials_file): - logger.info( - "GCP provider: Setting GOOGLE_APPLICATION_CREDENTIALS environment variable..." - ) - client_secrets_path = os.path.abspath(credentials_file) - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = client_secrets_path - - def print_credentials(self): - # Beautify audited profile, set "default" if there is no profile set - try: - getattr(self.session, "_service_account_email") - profile = ( - self.session._service_account_email - if self.session._service_account_email is not None - else "default" - ) - except AttributeError: - profile = "default" - - report = f""" -This report is being generated using credentials below: - -GCP Account: {Fore.YELLOW}[{profile}]{Style.RESET_ALL} GCP Project IDs: {Fore.YELLOW}[{", ".join(self.project_ids)}]{Style.RESET_ALL} -""" - print(report) - - def get_project_ids(self): - try: - project_ids = [] - - service = discovery.build( - "cloudresourcemanager", "v1", credentials=self.session - ) - - request = service.projects().list() - - while request is not None: - response = request.execute() - - for project in response.get("projects", []): - project_ids.append(project["projectId"]) - - request = service.projects().list_next( - previous_request=request, previous_response=response - ) - - return project_ids - except Exception as error: - logger.error( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - return [] diff --git a/prowler/providers/gcp/lib/service/service.py b/prowler/providers/gcp/lib/service/service.py index 581e0538ea..ffe7fcf141 100644 --- a/prowler/providers/gcp/lib/service/service.py +++ b/prowler/providers/gcp/lib/service/service.py @@ -8,7 +8,7 @@ from googleapiclient import discovery from googleapiclient.discovery import Resource from prowler.lib.logger import logger -from prowler.providers.gcp.gcp_provider_new import GcpProvider +from prowler.providers.gcp.gcp_provider import GcpProvider class GCPService: diff --git a/prowler/providers/gcp/services/apikeys/apikeys_service.py b/prowler/providers/gcp/services/apikeys/apikeys_service.py index e152d337cd..93e7b73fc6 100644 --- a/prowler/providers/gcp/services/apikeys/apikeys_service.py +++ b/prowler/providers/gcp/services/apikeys/apikeys_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## API Keys class APIKeys(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider, api_version="v2") self.keys = [] diff --git a/prowler/providers/gcp/services/bigquery/bigquery_service.py b/prowler/providers/gcp/services/bigquery/bigquery_service.py index ee7bbba625..3f338dfc0d 100644 --- a/prowler/providers/gcp/services/bigquery/bigquery_service.py +++ b/prowler/providers/gcp/services/bigquery/bigquery_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## BigQuery class BigQuery(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider, api_version="v2") self.datasets = [] diff --git a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py index 690cbe404e..e1c7416e28 100644 --- a/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py +++ b/prowler/providers/gcp/services/cloudresourcemanager/cloudresourcemanager_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## CloudResourceManager class CloudResourceManager(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.bindings = [] diff --git a/prowler/providers/gcp/services/cloudsql/cloudsql_service.py b/prowler/providers/gcp/services/cloudsql/cloudsql_service.py index 7b66cf4da1..4baf8a3b2b 100644 --- a/prowler/providers/gcp/services/cloudsql/cloudsql_service.py +++ b/prowler/providers/gcp/services/cloudsql/cloudsql_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## CloudSQL class CloudSQL(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__("sqladmin", provider) self.instances = [] self.__get_instances__() diff --git a/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py b/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py index 9e77541bf3..7c537260c7 100644 --- a/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py +++ b/prowler/providers/gcp/services/cloudstorage/cloudstorage_service.py @@ -3,12 +3,13 @@ from typing import Optional from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## CloudStorage class CloudStorage(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__("storage", provider) self.buckets = [] self.__get_buckets__() diff --git a/prowler/providers/gcp/services/compute/compute_service.py b/prowler/providers/gcp/services/compute/compute_service.py index fa23743e6c..59ce4c3957 100644 --- a/prowler/providers/gcp/services/compute/compute_service.py +++ b/prowler/providers/gcp/services/compute/compute_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## Compute class Compute(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.regions = set() self.zones = set() diff --git a/prowler/providers/gcp/services/dataproc/dataproc_service.py b/prowler/providers/gcp/services/dataproc/dataproc_service.py index 5eff989a39..fdcc4cfcd2 100644 --- a/prowler/providers/gcp/services/dataproc/dataproc_service.py +++ b/prowler/providers/gcp/services/dataproc/dataproc_service.py @@ -1,13 +1,14 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService from prowler.providers.gcp.services.compute.compute_client import compute_client ################## Dataproc class Dataproc(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.regions = compute_client.regions self.clusters = [] diff --git a/prowler/providers/gcp/services/dns/dns_service.py b/prowler/providers/gcp/services/dns/dns_service.py index 7082220a37..6e1d1d39d4 100644 --- a/prowler/providers/gcp/services/dns/dns_service.py +++ b/prowler/providers/gcp/services/dns/dns_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## DNS class DNS(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.managed_zones = [] self.__get_managed_zones__() diff --git a/prowler/providers/gcp/services/gke/gke_service.py b/prowler/providers/gcp/services/gke/gke_service.py index df3110ce0a..2137814569 100644 --- a/prowler/providers/gcp/services/gke/gke_service.py +++ b/prowler/providers/gcp/services/gke/gke_service.py @@ -1,13 +1,14 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## GKE class GKE(GCPService): - def __init__(self, provider): - super().__init__("container", audit_info, api_version="v1beta1") + def __init__(self, provider: GcpProvider): + super().__init__("container", provider, api_version="v1beta1") self.locations = [] self.__get_locations__() self.clusters = {} diff --git a/prowler/providers/gcp/services/iam/iam_service.py b/prowler/providers/gcp/services/iam/iam_service.py index 89d170c7a9..6e82ed8663 100644 --- a/prowler/providers/gcp/services/iam/iam_service.py +++ b/prowler/providers/gcp/services/iam/iam_service.py @@ -3,6 +3,7 @@ from datetime import datetime from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService from prowler.providers.gcp.services.cloudresourcemanager.cloudresourcemanager_client import ( cloudresourcemanager_client, @@ -11,7 +12,7 @@ from prowler.providers.gcp.services.cloudresourcemanager.cloudresourcemanager_cl ################## IAM class IAM(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.service_accounts = [] self.__get_service_accounts__() @@ -103,7 +104,7 @@ class ServiceAccount(BaseModel): ################## AccessApproval class AccessApproval(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.settings = {} self.__get_settings__() @@ -135,7 +136,7 @@ class Setting(BaseModel): ################## EssentialContacts class EssentialContacts(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.organizations = [] self.__get_contacts__() diff --git a/prowler/providers/gcp/services/kms/kms_service.py b/prowler/providers/gcp/services/kms/kms_service.py index f6f620c1bd..a736acce88 100644 --- a/prowler/providers/gcp/services/kms/kms_service.py +++ b/prowler/providers/gcp/services/kms/kms_service.py @@ -3,12 +3,13 @@ from typing import Optional from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## KMS class KMS(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__("cloudkms", provider) self.locations = [] self.key_rings = [] diff --git a/prowler/providers/gcp/services/logging/logging_service.py b/prowler/providers/gcp/services/logging/logging_service.py index bb93c3783a..311bdeecba 100644 --- a/prowler/providers/gcp/services/logging/logging_service.py +++ b/prowler/providers/gcp/services/logging/logging_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## Logging class Logging(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider, api_version="v2") self.sinks = [] self.metrics = [] diff --git a/prowler/providers/gcp/services/monitoring/monitoring_service.py b/prowler/providers/gcp/services/monitoring/monitoring_service.py index b605c90324..e0f364bf4a 100644 --- a/prowler/providers/gcp/services/monitoring/monitoring_service.py +++ b/prowler/providers/gcp/services/monitoring/monitoring_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## Monitoring class Monitoring(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider, api_version="v3") self.alert_policies = [] self.__get_alert_policies__() diff --git a/prowler/providers/gcp/services/serviceusage/serviceusage_service.py b/prowler/providers/gcp/services/serviceusage/serviceusage_service.py index d79ee8e0b8..726418c0fa 100644 --- a/prowler/providers/gcp/services/serviceusage/serviceusage_service.py +++ b/prowler/providers/gcp/services/serviceusage/serviceusage_service.py @@ -1,12 +1,13 @@ from pydantic import BaseModel from prowler.lib.logger import logger +from prowler.providers.gcp.gcp_provider import GcpProvider from prowler.providers.gcp.lib.service.service import GCPService ################## ServiceUsage class ServiceUsage(GCPService): - def __init__(self, provider): + def __init__(self, provider: GcpProvider): super().__init__(__class__.__name__, provider) self.active_services = {} self.__get_active_services__()