diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index a28b648618..9f43f4f4c9 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -14,6 +14,7 @@ All notable changes to the **Prowler SDK** are documented in this file. ### 🔄 Changed - Update Azure Monitor service metadata to new format [(#9622)](https://github.com/prowler-cloud/prowler/pull/9622) +- Parallelize Cloudflare zone API calls with threading to improve scan performance [(#9982)](https://github.com/prowler-cloud/prowler/pull/9982) ## [5.18.2] (Prowler UNRELEASED) diff --git a/prowler/providers/cloudflare/lib/service/service.py b/prowler/providers/cloudflare/lib/service/service.py index 8a1ea7484e..9eceef3592 100644 --- a/prowler/providers/cloudflare/lib/service/service.py +++ b/prowler/providers/cloudflare/lib/service/service.py @@ -1,5 +1,10 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed + +from prowler.lib.logger import logger from prowler.providers.cloudflare.cloudflare_provider import CloudflareProvider +MAX_WORKERS = 10 + class CloudflareService: """Base class for Cloudflare services to share provider context.""" @@ -10,3 +15,29 @@ class CloudflareService: self.audit_config = provider.audit_config self.fixer_config = provider.fixer_config self.service = service.lower() if not service.islower() else service + + # Thread pool for __threading_call__ + self.thread_pool = ThreadPoolExecutor(max_workers=MAX_WORKERS) + + def __threading_call__(self, call, iterator): + """Execute a function across multiple items using threading.""" + items = list(iterator) if not isinstance(iterator, list) else iterator + + futures = {self.thread_pool.submit(call, item): item for item in items} + results = [] + + for future in as_completed(futures): + try: + result = future.result() + if result is not None: + results.append(result) + except Exception as error: + # Log unhandled exceptions from threaded calls + item = futures[future] + item_id = getattr(item, "id", str(item)) + logger.error( + f"{self.service} - Threading error processing {item_id}: " + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + return results diff --git a/prowler/providers/cloudflare/services/zone/zone_service.py b/prowler/providers/cloudflare/services/zone/zone_service.py index 1fdb133e68..040bd59e92 100644 --- a/prowler/providers/cloudflare/services/zone/zone_service.py +++ b/prowler/providers/cloudflare/services/zone/zone_service.py @@ -49,13 +49,13 @@ class Zone(CloudflareService): super().__init__(__class__.__name__, provider) self.zones: dict[str, "CloudflareZone"] = {} self._list_zones() - self._get_zones_settings() - self._get_zones_dnssec() - self._get_zones_universal_ssl() - self._get_zones_rate_limit_rules() - self._get_zones_bot_management() - self._get_zones_firewall_rules() - self._get_zones_waf_rulesets() + self.__threading_call__(self._get_zone_settings_threaded, self.zones.values()) + self.__threading_call__(self._get_zone_dnssec, self.zones.values()) + self.__threading_call__(self._get_zone_universal_ssl, self.zones.values()) + self.__threading_call__(self._get_zone_rate_limit_rules, self.zones.values()) + self.__threading_call__(self._get_zone_bot_management, self.zones.values()) + self.__threading_call__(self._get_zone_firewall_rules, self.zones.values()) + self.__threading_call__(self._get_zone_waf_rulesets, self.zones.values()) def _list_zones(self) -> None: """List all Cloudflare zones with their basic information.""" @@ -124,110 +124,89 @@ class Zone(CloudflareService): f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def _get_zones_settings(self) -> None: - """Get settings for all zones.""" - logger.info("Zone - Getting zone settings...") - for zone in self.zones.values(): - try: - zone.settings = self._get_zone_settings(zone.id) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + def _get_zone_settings_threaded(self, zone: "CloudflareZone") -> None: + """Get settings for a single zone (thread-safe).""" + try: + zone.settings = self._get_zone_settings(zone.id) + except Exception as error: + logger.error( + f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) - def _get_zones_dnssec(self) -> None: - """Get DNSSEC status for all zones.""" - logger.info("Zone - Getting DNSSEC status...") - for zone in self.zones.values(): - try: - dnssec = self.client.dns.dnssec.get(zone_id=zone.id) - zone.dnssec_status = getattr(dnssec, "status", None) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + def _get_zone_dnssec(self, zone: "CloudflareZone") -> None: + """Get DNSSEC status for a single zone.""" + try: + dnssec = self.client.dns.dnssec.get(zone_id=zone.id) + zone.dnssec_status = getattr(dnssec, "status", None) + except Exception as error: + logger.error( + f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) - def _get_zones_universal_ssl(self) -> None: - """Get Universal SSL settings for all zones.""" - logger.info("Zone - Getting Universal SSL settings...") - for zone in self.zones.values(): - try: - universal_ssl = self.client.ssl.universal.settings.get(zone_id=zone.id) - zone.settings.universal_ssl_enabled = getattr( - universal_ssl, "enabled", False - ) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + def _get_zone_universal_ssl(self, zone: "CloudflareZone") -> None: + """Get Universal SSL settings for a single zone.""" + try: + universal_ssl = self.client.ssl.universal.settings.get(zone_id=zone.id) + zone.settings.universal_ssl_enabled = getattr( + universal_ssl, "enabled", False + ) + except Exception as error: + logger.error( + f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) - def _get_zones_rate_limit_rules(self) -> None: - """Get rate limiting rules for all zones.""" - logger.info("Zone - Getting rate limit rules...") - for zone in self.zones.values(): - try: - seen_ruleset_ids: set[str] = set() - for ruleset in self.client.rulesets.list(zone_id=zone.id): - ruleset_id = getattr(ruleset, "id", "") - if ruleset_id in seen_ruleset_ids: - break - seen_ruleset_ids.add(ruleset_id) + def _get_zone_rate_limit_rules(self, zone: "CloudflareZone") -> None: + """Get rate limiting rules for a single zone.""" + try: + seen_ruleset_ids: set[str] = set() + for ruleset in self.client.rulesets.list(zone_id=zone.id): + ruleset_id = getattr(ruleset, "id", "") + if ruleset_id in seen_ruleset_ids: + break + seen_ruleset_ids.add(ruleset_id) - phase = getattr(ruleset, "phase", "") - if phase == "http_ratelimit": - try: - ruleset_detail = self.client.rulesets.get( - ruleset_id=ruleset_id, zone_id=zone.id - ) - rules = getattr(ruleset_detail, "rules", []) or [] - seen_rule_ids: set[str] = set() - for rule in rules: - rule_id = getattr(rule, "id", "") - if rule_id in seen_rule_ids: - break - seen_rule_ids.add(rule_id) - zone.rate_limit_rules.append( - CloudflareRateLimitRule( - id=rule_id, - description=getattr(rule, "description", None), - action=getattr(rule, "action", None), - enabled=getattr(rule, "enabled", True), - expression=getattr(rule, "expression", None), - ) + phase = getattr(ruleset, "phase", "") + if phase == "http_ratelimit": + try: + ruleset_detail = self.client.rulesets.get( + ruleset_id=ruleset_id, zone_id=zone.id + ) + rules = getattr(ruleset_detail, "rules", []) or [] + seen_rule_ids: set[str] = set() + for rule in rules: + rule_id = getattr(rule, "id", "") + if rule_id in seen_rule_ids: + break + seen_rule_ids.add(rule_id) + zone.rate_limit_rules.append( + CloudflareRateLimitRule( + id=rule_id, + description=getattr(rule, "description", None), + action=getattr(rule, "action", None), + enabled=getattr(rule, "enabled", True), + expression=getattr(rule, "expression", None), ) - except Exception as error: - logger.debug( - f"{zone.id} ruleset {ruleset_id} -- {error.__class__.__name__}: {error}" ) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + except Exception as error: + logger.debug( + f"{zone.id} ruleset {ruleset_id} -- {error.__class__.__name__}: {error}" + ) + except Exception as error: + logger.error( + f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) - def _get_zones_bot_management(self) -> None: - """Get Bot Management settings for all zones.""" - logger.info("Zone - Getting Bot Management settings...") - for zone in self.zones.values(): - try: - bot_management = self.client.bot_management.get(zone_id=zone.id) - zone.settings.bot_fight_mode_enabled = getattr( - bot_management, "fight_mode", False - ) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - - def _get_zones_firewall_rules(self) -> None: - """Get firewall rules for all zones.""" - logger.info("Zone - Getting firewall rules...") - for zone in self.zones.values(): - try: - self._get_zone_firewall_rules(zone) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) + def _get_zone_bot_management(self, zone: "CloudflareZone") -> None: + """Get Bot Management settings for a single zone.""" + try: + bot_management = self.client.bot_management.get(zone_id=zone.id) + zone.settings.bot_fight_mode_enabled = getattr( + bot_management, "fight_mode", False + ) + except Exception as error: + logger.error( + f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) def _get_zone_firewall_rules(self, zone: "CloudflareZone") -> None: """List firewall rules from custom rulesets for a zone.""" @@ -282,17 +261,6 @@ class Zone(CloudflareService): f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" ) - def _get_zones_waf_rulesets(self) -> None: - """Get WAF rulesets for all zones.""" - logger.info("Zone - Getting WAF rulesets...") - for zone in self.zones.values(): - try: - self._get_zone_waf_rulesets(zone) - except Exception as error: - logger.error( - f"{zone.id} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - def _get_zone_waf_rulesets(self, zone: "CloudflareZone") -> None: """List WAF rulesets for a zone using the rulesets API.""" seen_ids: set[str] = set()