From 26a00a14dfc6286db633b67d7debf59de858196b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Jes=C3=BAs=20Pe=C3=B1a=20Rodr=C3=ADguez?= Date: Thu, 17 Oct 2024 19:07:18 +0200 Subject: [PATCH] feat(datasync): add datasync service and check `datasync_task_logging_enabled` (#5444) --- .../aws/services/datasync/__init__.py | 0 .../aws/services/datasync/datasync_client.py | 4 + .../aws/services/datasync/datasync_service.py | 118 +++++++++++ .../datasync_task_logging_enabled/__init__.py | 0 ...atasync_task_logging_enabled.metadata.json | 34 +++ .../datasync_task_logging_enabled.py | 40 ++++ .../datasync/datasync_service_test.py | 197 ++++++++++++++++++ .../datasync_task_logging_enabled_test.py | 127 +++++++++++ 8 files changed, 520 insertions(+) create mode 100644 prowler/providers/aws/services/datasync/__init__.py create mode 100644 prowler/providers/aws/services/datasync/datasync_client.py create mode 100644 prowler/providers/aws/services/datasync/datasync_service.py create mode 100644 prowler/providers/aws/services/datasync/datasync_task_logging_enabled/__init__.py create mode 100644 prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.metadata.json create mode 100644 prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.py create mode 100644 tests/providers/aws/services/datasync/datasync_service_test.py create mode 100644 tests/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled_test.py diff --git a/prowler/providers/aws/services/datasync/__init__.py b/prowler/providers/aws/services/datasync/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/aws/services/datasync/datasync_client.py b/prowler/providers/aws/services/datasync/datasync_client.py new file mode 100644 index 0000000000..f8b9c0e688 --- /dev/null +++ b/prowler/providers/aws/services/datasync/datasync_client.py @@ -0,0 +1,4 @@ +from prowler.providers.aws.services.datasync.datasync_service import DataSync +from prowler.providers.common.provider import Provider + +datasync_client = DataSync(Provider.get_global_provider()) diff --git a/prowler/providers/aws/services/datasync/datasync_service.py b/prowler/providers/aws/services/datasync/datasync_service.py new file mode 100644 index 0000000000..9a75855d75 --- /dev/null +++ b/prowler/providers/aws/services/datasync/datasync_service.py @@ -0,0 +1,118 @@ +from typing import Dict, List, Optional + +from botocore.exceptions import ClientError +from pydantic import BaseModel, Field + +from prowler.lib.logger import logger +from prowler.lib.scan_filters.scan_filters import is_resource_filtered +from prowler.providers.aws.lib.service.service import AWSService + + +class DataSync(AWSService): + """AWS DataSync service class to list tasks, describe them, and list their tags.""" + + def __init__(self, provider): + """Initialize the DataSync service. + + Args: + provider: The AWS provider instance. + """ + + super().__init__(__class__.__name__, provider) + self.tasks = {} + self.__threading_call__(self._list_tasks) + self.__threading_call__(self._describe_tasks, self.tasks.values()) + self.__threading_call__(self._list_task_tags, self.tasks.values()) + + def _list_tasks(self, regional_client): + """List DataSync tasks in the given region. + + Args: + regional_client: The regional AWS client. + """ + + logger.info("DataSync - Listing tasks...") + try: + list_tasks_paginator = regional_client.get_paginator("list_tasks") + for page in list_tasks_paginator.paginate(): + for task in page.get("Tasks", []): + task_arn = task["TaskArn"] + task_id = task_arn.split("/")[-1] + if not self.audit_resources or ( + is_resource_filtered(task_arn, self.audit_resources) + ): + self.tasks[task_arn] = DataSyncTask( + id=task_id, + arn=task_arn, + name=task.get("Name"), + region=regional_client.region, + ) + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + def _describe_tasks(self, task): + """Describe each DataSync task and update task details.""" + + logger.info("DataSync - Describing tasks...") + try: + regional_client = self.regional_clients[task.region] + response = regional_client.describe_task(TaskArn=task.arn) + task.status = response.get("Status") + task.options = response.get("Options") + task.source_location_arn = response.get("SourceLocationArn") + task.destination_location_arn = response.get("DestinationLocationArn") + task.excludes = response.get("Excludes") + task.schedule = response.get("Schedule") + task.cloudwatch_log_group_arn = response.get("CloudWatchLogGroupArn") + except ClientError as error: + if error.response["Error"]["Code"] == "ResourceNotFoundException": + logger.warning( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + else: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + def _list_task_tags(self, task): + """List tags for each DataSync task.""" + + logger.info("DataSync - Listing task tags...") + try: + regional_client = self.regional_clients[task.region] + response = regional_client.list_tags_for_resource(ResourceArn=task.arn) + task.tags = response.get("Tags", []) + except ClientError as error: + if error.response["Error"]["Code"] == "ResourceNotFoundException": + logger.warning( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + else: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + except Exception as error: + logger.error( + f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + + +class DataSyncTask(BaseModel): + id: str + name: Optional[str] = None + arn: str + region: str + status: Optional[str] = None + options: Optional[Dict] = None + source_location_arn: Optional[str] = None + destination_location_arn: Optional[str] = None + excludes: Optional[List] = None + schedule: Optional[Dict] = None + cloudwatch_log_group_arn: Optional[str] = None + tags: List[Dict] = Field(default_factory=list) diff --git a/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/__init__.py b/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.metadata.json b/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.metadata.json new file mode 100644 index 0000000000..736c6fa9de --- /dev/null +++ b/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.metadata.json @@ -0,0 +1,34 @@ +{ + "Provider": "aws", + "CheckID": "datasync_task_logging_enabled", + "CheckTitle": "DataSync tasks should have logging enabled", + "CheckType": [ + "Software and Configuration Checks/AWS Security Best Practices" + ], + "ServiceName": "datasync", + "SubServiceName": "", + "ResourceIdTemplate": "arn:aws:datasync:{region}:{account-id}:task/{task-id}", + "Severity": "high", + "ResourceType": "AwsDataSyncTask", + "Description": "This control checks if AWS DataSync tasks have logging enabled. The control fails if the task doesn't have the CloudWatchLogGroupArn property defined.", + "Risk": "Without logging enabled, important operational data may be lost, making it difficult to troubleshoot issues, monitor performance, and ensure compliance with auditing requirements.", + "RelatedUrl": "https://docs.aws.amazon.com/datasync/latest/userguide/monitor-datasync.html#enable-logging", + "Remediation": { + "Code": { + "CLI": "aws datasync update-task --task-arn --cloud-watch-log-group-arn ", + "NativeIaC": "", + "Other": "https://docs.aws.amazon.com/datasync/latest/userguide/monitor-datasync.html#enable-logging", + "Terraform": "" + }, + "Recommendation": { + "Text": "Configure logging for your DataSync tasks to ensure that operational data is captured and available for debugging, monitoring, and auditing purposes.", + "Url": "https://docs.aws.amazon.com/datasync/latest/userguide/monitor-datasync.html#enable-logging" + } + }, + "Categories": [ + "logging" + ], + "DependsOn": [], + "RelatedTo": [], + "Notes": "" +} diff --git a/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.py b/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.py new file mode 100644 index 0000000000..a00313bdda --- /dev/null +++ b/prowler/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled.py @@ -0,0 +1,40 @@ +from typing import List + +from prowler.lib.check.models import Check, Check_Report_AWS +from prowler.providers.aws.services.datasync.datasync_client import datasync_client + + +class datasync_task_logging_enabled(Check): + """Check if AWS DataSync tasks have logging enabled. + + This class verifies whether each AWS DataSync task has logging enabled by checking + for the presence of a CloudWatch Log Group ARN in the task's configuration. + """ + + def execute(self) -> List[Check_Report_AWS]: + """Execute the DataSync tasks logging enabled check. + + Iterates over all DataSync tasks and generates a report indicating whether + each task has logging enabled. + + Returns: + List[Check_Report_AWS]: A list of report objects with the results of the check. + """ + findings = [] + for task in datasync_client.tasks.values(): + report = Check_Report_AWS(self.metadata()) + report.region = task.region + report.resource_id = task.id + report.resource_arn = task.arn + report.resource_tags = task.tags + report.status = "PASS" + report.status_extended = f"DataSync task {task.name} has logging enabled." + + if not task.cloudwatch_log_group_arn: + report.status = "FAIL" + report.status_extended = ( + f"DataSync task {task.name} does not have logging enabled." + ) + + findings.append(report) + return findings diff --git a/tests/providers/aws/services/datasync/datasync_service_test.py b/tests/providers/aws/services/datasync/datasync_service_test.py new file mode 100644 index 0000000000..13fc22dc88 --- /dev/null +++ b/tests/providers/aws/services/datasync/datasync_service_test.py @@ -0,0 +1,197 @@ +from unittest.mock import MagicMock, patch + +import botocore +from botocore.exceptions import ClientError + +from prowler.providers.aws.services.datasync.datasync_service import DataSync +from tests.providers.aws.utils import AWS_REGION_EU_WEST_1, set_mocked_aws_provider + +make_api_call = botocore.client.BaseClient._make_api_call + + +def mock_make_api_call(self, operation_name, kwarg): + # Simulate ResourceNotFoundException for specific ARNs + if operation_name in ["DescribeTask", "ListTagsForResource"]: + if "not-found" in kwarg.get("TaskArn", "") or "not-found" in kwarg.get( + "ResourceArn", "" + ): + raise ClientError( + { + "Error": { + "Code": "ResourceNotFoundException", + "Message": "Resource not found", + } + }, + operation_name, + ) + # Simulate other ClientError + if "client-error" in kwarg.get("TaskArn", "") or "client-error" in kwarg.get( + "ResourceArn", "" + ): + raise ClientError( + { + "Error": { + "Code": "InternalServerError", + "Message": "Internal server error", + } + }, + operation_name, + ) + # Simulate generic exception + if "generic-error" in kwarg.get("TaskArn", "") or "generic-error" in kwarg.get( + "ResourceArn", "" + ): + raise Exception("Generic error") + + if operation_name == "ListTasks": + if kwarg.get("generic_error", False): + raise Exception("Generic error in ListTasks") + return { + "Tasks": [ + { + "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/task-12345678901234567", + "Name": "test_task", + }, + { + "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/not-found", + "Name": "not_found_task", + }, + { + "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/client-error", + "Name": "client_error_task", + }, + { + "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/generic-error", + "Name": "generic_error_task", + }, + ] + } + if operation_name == "DescribeTask": + return { + "TaskArn": kwarg["TaskArn"], + "Status": "AVAILABLE", + "Name": "test_task", + "CurrentTaskExecutionArn": "arn:aws:datasync:eu-west-1:123456789012:task/task-12345678901234567/execution/exec-12345678901234567", + "Options": {}, + "SourceLocationArn": "arn:aws:datasync:eu-west-1:123456789012:location/loc-12345678901234567", + "DestinationLocationArn": "arn:aws:datasync:eu-west-1:123456789012:location/loc-76543210987654321", + "CloudWatchLogGroupArn": "arn:aws:logs:eu-west-1:123456789012:log-group:/aws/datasync/log-group", + "Tags": [ + {"Key": "Name", "Value": "test_task"}, + ], + } + + if operation_name == "ListTagsForResource": + return { + "Tags": [ + {"Key": "Name", "Value": "test_task"}, + ], + } + + return make_api_call(self, operation_name, kwarg) + + +def mock_generate_regional_clients(provider, service): + regional_client = provider._session.current_session.client( + service, region_name=AWS_REGION_EU_WEST_1 + ) + regional_client.region = AWS_REGION_EU_WEST_1 + return {AWS_REGION_EU_WEST_1: regional_client} + + +@patch( + "prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients", + new=mock_generate_regional_clients, +) +class Test_DataSync_Service: + # Test DataSync Service initialization + def test_service(self): + aws_provider = set_mocked_aws_provider() + datasync = DataSync(aws_provider) + assert datasync.service == "datasync" + + # Test DataSync clients creation + def test_client(self): + aws_provider = set_mocked_aws_provider() + datasync = DataSync(aws_provider) + for reg_client in datasync.regional_clients.values(): + assert reg_client.__class__.__name__ == "DataSync" + + # Test DataSync session + def test__get_session__(self): + aws_provider = set_mocked_aws_provider() + datasync = DataSync(aws_provider) + assert datasync.session.__class__.__name__ == "Session" + + # Test listing DataSync tasks + @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call) + def test_list_tasks(self): + aws_provider = set_mocked_aws_provider() + datasync = DataSync(aws_provider) + + task_arn = "arn:aws:datasync:eu-west-1:123456789012:task/task-12345678901234567" + found_task = None + for task in datasync.tasks.values(): + if task.arn == task_arn: + found_task = task + break + + assert found_task + assert found_task.name == "test_task" + assert found_task.region == AWS_REGION_EU_WEST_1 + + # Test generic exception in list_tasks + def test_list_tasks_generic_exception(self): + aws_provider = set_mocked_aws_provider() + + # Mock the regional client's list_tasks method specifically + mock_client = MagicMock() + mock_client.region = AWS_REGION_EU_WEST_1 + mock_client.get_paginator.side_effect = Exception("Generic error in ListTasks") + + datasync = DataSync(aws_provider) + assert len(datasync.tasks.values()) == 0 + + # Test describing DataSync tasks with various exceptions + @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call) + def test_describe_tasks_with_exceptions(self): + aws_provider = set_mocked_aws_provider() + datasync = DataSync(aws_provider) + + # Check all tasks were processed despite exceptions + assert len(datasync.tasks.values()) == 4 + + # Verify each task type + tasks_by_name = {task.name: task for task in datasync.tasks.values()} + + # Normal task + assert "test_task" in tasks_by_name + assert tasks_by_name["test_task"].status == "AVAILABLE" + + # ResourceNotFoundException task + assert "not_found_task" in tasks_by_name + assert not tasks_by_name["not_found_task"].status + + # ClientError task + assert "client_error_task" in tasks_by_name + assert not tasks_by_name["client_error_task"].status + + # Generic error task + assert "generic_error_task" in tasks_by_name + assert not tasks_by_name["generic_error_task"].status + + # Test listing task tags with various exceptions + @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call) + def test_list_task_tags_with_exceptions(self): + aws_provider = set_mocked_aws_provider() + datasync = DataSync(aws_provider) + + tasks_by_name = {task.name: task for task in datasync.tasks.values()} + assert tasks_by_name["test_task"].tags == [ + {"Key": "Name", "Value": "test_task"} + ] + + # Tasks with exceptions should have empty tag lists + assert tasks_by_name["not_found_task"].tags == [] + assert tasks_by_name["client_error_task"].tags == [] + assert tasks_by_name["generic_error_task"].tags == [] diff --git a/tests/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled_test.py b/tests/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled_test.py new file mode 100644 index 0000000000..fd79aad29e --- /dev/null +++ b/tests/providers/aws/services/datasync/datasync_task_logging_enabled/datasync_task_logging_enabled_test.py @@ -0,0 +1,127 @@ +from unittest.mock import patch + +from tests.providers.aws.utils import AWS_REGION_US_EAST_1, set_mocked_aws_provider + +TASK_ID = "task-12345" +TASK_ARN = f"arn:aws:datasync:{AWS_REGION_US_EAST_1}:123456789012:task/{TASK_ID}" + + +class Test_datasync_task_logging_enabled: + def test_no_tasks(self): + from prowler.providers.aws.services.datasync.datasync_service import DataSync + + # Set up a mocked AWS provider + mocked_aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1]) + + # Create a DataSync client with no tasks + datasync_client = DataSync(mocked_aws_provider) + datasync_client.tasks = {} + + with patch( + "prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled.datasync_client", + new=datasync_client, + ), patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=mocked_aws_provider, + ): + from prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled import ( + datasync_task_logging_enabled, + ) + + check = datasync_task_logging_enabled() + result = check.execute() + assert len(result) == 0 + + def test_task_without_logging(self): + from prowler.providers.aws.services.datasync.datasync_service import ( + DataSync, + DataSyncTask, + ) + + # Set up a mocked AWS provider + mocked_aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1]) + + # Create a DataSync task without logging enabled + task = DataSyncTask( + id=TASK_ID, + arn=TASK_ARN, + name="TestTask", + region=AWS_REGION_US_EAST_1, + cloudwatch_log_group_arn=None, # Logging not enabled + tags=[], + ) + + # Create a DataSync client with the task + datasync_client = DataSync(mocked_aws_provider) + datasync_client.tasks[TASK_ARN] = task + + with patch( + "prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled.datasync_client", + new=datasync_client, + ), patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=mocked_aws_provider, + ): + from prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled import ( + datasync_task_logging_enabled, + ) + + check = datasync_task_logging_enabled() + result = check.execute() + assert len(result) == 1 + assert result[0].status == "FAIL" + assert ( + result[0].status_extended + == f"DataSync task {task.name} does not have logging enabled." + ) + assert result[0].resource_id == TASK_ID + assert result[0].resource_arn == TASK_ARN + assert result[0].region == AWS_REGION_US_EAST_1 + assert result[0].resource_tags == [] + + def test_task_with_logging(self): + from prowler.providers.aws.services.datasync.datasync_service import ( + DataSync, + DataSyncTask, + ) + + # Set up a mocked AWS provider + mocked_aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1]) + + # Create a DataSync task with logging enabled + task = DataSyncTask( + id=TASK_ID, + arn=TASK_ARN, + name="TestTask", + region=AWS_REGION_US_EAST_1, + cloudwatch_log_group_arn=f"arn:aws:logs:{AWS_REGION_US_EAST_1}:123456789012:log-group:datasync-log-group", + tags=[], + ) + + # Create a DataSync client with the task + datasync_client = DataSync(mocked_aws_provider) + datasync_client.tasks[TASK_ARN] = task + + with patch( + "prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled.datasync_client", + new=datasync_client, + ), patch( + "prowler.providers.common.provider.Provider.get_global_provider", + return_value=mocked_aws_provider, + ): + from prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled import ( + datasync_task_logging_enabled, + ) + + check = datasync_task_logging_enabled() + result = check.execute() + assert len(result) == 1 + assert result[0].status == "PASS" + assert ( + result[0].status_extended + == f"DataSync task {task.name} has logging enabled." + ) + assert result[0].resource_id == TASK_ID + assert result[0].resource_arn == TASK_ARN + assert result[0].region == AWS_REGION_US_EAST_1 + assert result[0].resource_tags == []