feat(datasync): add datasync service and check datasync_task_logging_enabled (#5444)

2026-07-17 17:41:50 +00:00 · 2024-10-17 19:07:18 +02:00
parent 12abea371d
commit 26a00a14df
8 changed files with 520 additions and 0 deletions
@@ -0,0 +1,4 @@
+from prowler.providers.aws.services.datasync.datasync_service import DataSync
+from prowler.providers.common.provider import Provider
+
+datasync_client = DataSync(Provider.get_global_provider())
@@ -0,0 +1,118 @@
+from typing import Dict, List, Optional
+
+from botocore.exceptions import ClientError
+from pydantic import BaseModel, Field
+
+from prowler.lib.logger import logger
+from prowler.lib.scan_filters.scan_filters import is_resource_filtered
+from prowler.providers.aws.lib.service.service import AWSService
+
+
+class DataSync(AWSService):
+    """AWS DataSync service class to list tasks, describe them, and list their tags."""
+
+    def __init__(self, provider):
+        """Initialize the DataSync service.
+
+        Args:
+            provider: The AWS provider instance.
+        """
+
+        super().__init__(__class__.__name__, provider)
+        self.tasks = {}
+        self.__threading_call__(self._list_tasks)
+        self.__threading_call__(self._describe_tasks, self.tasks.values())
+        self.__threading_call__(self._list_task_tags, self.tasks.values())
+
+    def _list_tasks(self, regional_client):
+        """List DataSync tasks in the given region.
+
+        Args:
+            regional_client: The regional AWS client.
+        """
+
+        logger.info("DataSync - Listing tasks...")
+        try:
+            list_tasks_paginator = regional_client.get_paginator("list_tasks")
+            for page in list_tasks_paginator.paginate():
+                for task in page.get("Tasks", []):
+                    task_arn = task["TaskArn"]
+                    task_id = task_arn.split("/")[-1]
+                    if not self.audit_resources or (
+                        is_resource_filtered(task_arn, self.audit_resources)
+                    ):
+                        self.tasks[task_arn] = DataSyncTask(
+                            id=task_id,
+                            arn=task_arn,
+                            name=task.get("Name"),
+                            region=regional_client.region,
+                        )
+        except Exception as error:
+            logger.error(
+                f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+            )
+
+    def _describe_tasks(self, task):
+        """Describe each DataSync task and update task details."""
+
+        logger.info("DataSync - Describing tasks...")
+        try:
+            regional_client = self.regional_clients[task.region]
+            response = regional_client.describe_task(TaskArn=task.arn)
+            task.status = response.get("Status")
+            task.options = response.get("Options")
+            task.source_location_arn = response.get("SourceLocationArn")
+            task.destination_location_arn = response.get("DestinationLocationArn")
+            task.excludes = response.get("Excludes")
+            task.schedule = response.get("Schedule")
+            task.cloudwatch_log_group_arn = response.get("CloudWatchLogGroupArn")
+        except ClientError as error:
+            if error.response["Error"]["Code"] == "ResourceNotFoundException":
+                logger.warning(
+                    f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+                )
+            else:
+                logger.error(
+                    f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+                )
+        except Exception as error:
+            logger.error(
+                f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+            )
+
+    def _list_task_tags(self, task):
+        """List tags for each DataSync task."""
+
+        logger.info("DataSync - Listing task tags...")
+        try:
+            regional_client = self.regional_clients[task.region]
+            response = regional_client.list_tags_for_resource(ResourceArn=task.arn)
+            task.tags = response.get("Tags", [])
+        except ClientError as error:
+            if error.response["Error"]["Code"] == "ResourceNotFoundException":
+                logger.warning(
+                    f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+                )
+            else:
+                logger.error(
+                    f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+                )
+        except Exception as error:
+            logger.error(
+                f"{regional_client.region} -- {error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}"
+            )
+
+
+class DataSyncTask(BaseModel):
+    id: str
+    name: Optional[str] = None
+    arn: str
+    region: str
+    status: Optional[str] = None
+    options: Optional[Dict] = None
+    source_location_arn: Optional[str] = None
+    destination_location_arn: Optional[str] = None
+    excludes: Optional[List] = None
+    schedule: Optional[Dict] = None
+    cloudwatch_log_group_arn: Optional[str] = None
+    tags: List[Dict] = Field(default_factory=list)
@@ -0,0 +1,34 @@
+{
+  "Provider": "aws",
+  "CheckID": "datasync_task_logging_enabled",
+  "CheckTitle": "DataSync tasks should have logging enabled",
+  "CheckType": [
+    "Software and Configuration Checks/AWS Security Best Practices"
+  ],
+  "ServiceName": "datasync",
+  "SubServiceName": "",
+  "ResourceIdTemplate": "arn:aws:datasync:{region}:{account-id}:task/{task-id}",
+  "Severity": "high",
+  "ResourceType": "AwsDataSyncTask",
+  "Description": "This control checks if AWS DataSync tasks have logging enabled. The control fails if the task doesn't have the CloudWatchLogGroupArn property defined.",
+  "Risk": "Without logging enabled, important operational data may be lost, making it difficult to troubleshoot issues, monitor performance, and ensure compliance with auditing requirements.",
+  "RelatedUrl": "https://docs.aws.amazon.com/datasync/latest/userguide/monitor-datasync.html#enable-logging",
+  "Remediation": {
+    "Code": {
+      "CLI": "aws datasync update-task --task-arn <task-arn> --cloud-watch-log-group-arn <log-group-arn>",
+      "NativeIaC": "",
+      "Other": "https://docs.aws.amazon.com/datasync/latest/userguide/monitor-datasync.html#enable-logging",
+      "Terraform": ""
+    },
+    "Recommendation": {
+      "Text": "Configure logging for your DataSync tasks to ensure that operational data is captured and available for debugging, monitoring, and auditing purposes.",
+      "Url": "https://docs.aws.amazon.com/datasync/latest/userguide/monitor-datasync.html#enable-logging"
+    }
+  },
+  "Categories": [
+    "logging"
+  ],
+  "DependsOn": [],
+  "RelatedTo": [],
+  "Notes": ""
+}
@@ -0,0 +1,40 @@
+from typing import List
+
+from prowler.lib.check.models import Check, Check_Report_AWS
+from prowler.providers.aws.services.datasync.datasync_client import datasync_client
+
+
+class datasync_task_logging_enabled(Check):
+    """Check if AWS DataSync tasks have logging enabled.
+
+    This class verifies whether each AWS DataSync task has logging enabled by checking
+    for the presence of a CloudWatch Log Group ARN in the task's configuration.
+    """
+
+    def execute(self) -> List[Check_Report_AWS]:
+        """Execute the DataSync tasks logging enabled check.
+
+        Iterates over all DataSync tasks and generates a report indicating whether
+        each task has logging enabled.
+
+        Returns:
+            List[Check_Report_AWS]: A list of report objects with the results of the check.
+        """
+        findings = []
+        for task in datasync_client.tasks.values():
+            report = Check_Report_AWS(self.metadata())
+            report.region = task.region
+            report.resource_id = task.id
+            report.resource_arn = task.arn
+            report.resource_tags = task.tags
+            report.status = "PASS"
+            report.status_extended = f"DataSync task {task.name} has logging enabled."
+
+            if not task.cloudwatch_log_group_arn:
+                report.status = "FAIL"
+                report.status_extended = (
+                    f"DataSync task {task.name} does not have logging enabled."
+                )
+
+            findings.append(report)
+        return findings
@@ -0,0 +1,197 @@
+from unittest.mock import MagicMock, patch
+
+import botocore
+from botocore.exceptions import ClientError
+
+from prowler.providers.aws.services.datasync.datasync_service import DataSync
+from tests.providers.aws.utils import AWS_REGION_EU_WEST_1, set_mocked_aws_provider
+
+make_api_call = botocore.client.BaseClient._make_api_call
+
+
+def mock_make_api_call(self, operation_name, kwarg):
+    # Simulate ResourceNotFoundException for specific ARNs
+    if operation_name in ["DescribeTask", "ListTagsForResource"]:
+        if "not-found" in kwarg.get("TaskArn", "") or "not-found" in kwarg.get(
+            "ResourceArn", ""
+        ):
+            raise ClientError(
+                {
+                    "Error": {
+                        "Code": "ResourceNotFoundException",
+                        "Message": "Resource not found",
+                    }
+                },
+                operation_name,
+            )
+        # Simulate other ClientError
+        if "client-error" in kwarg.get("TaskArn", "") or "client-error" in kwarg.get(
+            "ResourceArn", ""
+        ):
+            raise ClientError(
+                {
+                    "Error": {
+                        "Code": "InternalServerError",
+                        "Message": "Internal server error",
+                    }
+                },
+                operation_name,
+            )
+        # Simulate generic exception
+        if "generic-error" in kwarg.get("TaskArn", "") or "generic-error" in kwarg.get(
+            "ResourceArn", ""
+        ):
+            raise Exception("Generic error")
+
+    if operation_name == "ListTasks":
+        if kwarg.get("generic_error", False):
+            raise Exception("Generic error in ListTasks")
+        return {
+            "Tasks": [
+                {
+                    "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/task-12345678901234567",
+                    "Name": "test_task",
+                },
+                {
+                    "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/not-found",
+                    "Name": "not_found_task",
+                },
+                {
+                    "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/client-error",
+                    "Name": "client_error_task",
+                },
+                {
+                    "TaskArn": "arn:aws:datasync:eu-west-1:123456789012:task/generic-error",
+                    "Name": "generic_error_task",
+                },
+            ]
+        }
+    if operation_name == "DescribeTask":
+        return {
+            "TaskArn": kwarg["TaskArn"],
+            "Status": "AVAILABLE",
+            "Name": "test_task",
+            "CurrentTaskExecutionArn": "arn:aws:datasync:eu-west-1:123456789012:task/task-12345678901234567/execution/exec-12345678901234567",
+            "Options": {},
+            "SourceLocationArn": "arn:aws:datasync:eu-west-1:123456789012:location/loc-12345678901234567",
+            "DestinationLocationArn": "arn:aws:datasync:eu-west-1:123456789012:location/loc-76543210987654321",
+            "CloudWatchLogGroupArn": "arn:aws:logs:eu-west-1:123456789012:log-group:/aws/datasync/log-group",
+            "Tags": [
+                {"Key": "Name", "Value": "test_task"},
+            ],
+        }
+
+    if operation_name == "ListTagsForResource":
+        return {
+            "Tags": [
+                {"Key": "Name", "Value": "test_task"},
+            ],
+        }
+
+    return make_api_call(self, operation_name, kwarg)
+
+
+def mock_generate_regional_clients(provider, service):
+    regional_client = provider._session.current_session.client(
+        service, region_name=AWS_REGION_EU_WEST_1
+    )
+    regional_client.region = AWS_REGION_EU_WEST_1
+    return {AWS_REGION_EU_WEST_1: regional_client}
+
+
+@patch(
+    "prowler.providers.aws.aws_provider.AwsProvider.generate_regional_clients",
+    new=mock_generate_regional_clients,
+)
+class Test_DataSync_Service:
+    # Test DataSync Service initialization
+    def test_service(self):
+        aws_provider = set_mocked_aws_provider()
+        datasync = DataSync(aws_provider)
+        assert datasync.service == "datasync"
+
+    # Test DataSync clients creation
+    def test_client(self):
+        aws_provider = set_mocked_aws_provider()
+        datasync = DataSync(aws_provider)
+        for reg_client in datasync.regional_clients.values():
+            assert reg_client.__class__.__name__ == "DataSync"
+
+    # Test DataSync session
+    def test__get_session__(self):
+        aws_provider = set_mocked_aws_provider()
+        datasync = DataSync(aws_provider)
+        assert datasync.session.__class__.__name__ == "Session"
+
+    # Test listing DataSync tasks
+    @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
+    def test_list_tasks(self):
+        aws_provider = set_mocked_aws_provider()
+        datasync = DataSync(aws_provider)
+
+        task_arn = "arn:aws:datasync:eu-west-1:123456789012:task/task-12345678901234567"
+        found_task = None
+        for task in datasync.tasks.values():
+            if task.arn == task_arn:
+                found_task = task
+                break
+
+        assert found_task
+        assert found_task.name == "test_task"
+        assert found_task.region == AWS_REGION_EU_WEST_1
+
+    # Test generic exception in list_tasks
+    def test_list_tasks_generic_exception(self):
+        aws_provider = set_mocked_aws_provider()
+
+        # Mock the regional client's list_tasks method specifically
+        mock_client = MagicMock()
+        mock_client.region = AWS_REGION_EU_WEST_1
+        mock_client.get_paginator.side_effect = Exception("Generic error in ListTasks")
+
+        datasync = DataSync(aws_provider)
+        assert len(datasync.tasks.values()) == 0
+
+    # Test describing DataSync tasks with various exceptions
+    @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
+    def test_describe_tasks_with_exceptions(self):
+        aws_provider = set_mocked_aws_provider()
+        datasync = DataSync(aws_provider)
+
+        # Check all tasks were processed despite exceptions
+        assert len(datasync.tasks.values()) == 4
+
+        # Verify each task type
+        tasks_by_name = {task.name: task for task in datasync.tasks.values()}
+
+        # Normal task
+        assert "test_task" in tasks_by_name
+        assert tasks_by_name["test_task"].status == "AVAILABLE"
+
+        # ResourceNotFoundException task
+        assert "not_found_task" in tasks_by_name
+        assert not tasks_by_name["not_found_task"].status
+
+        # ClientError task
+        assert "client_error_task" in tasks_by_name
+        assert not tasks_by_name["client_error_task"].status
+
+        # Generic error task
+        assert "generic_error_task" in tasks_by_name
+        assert not tasks_by_name["generic_error_task"].status
+
+    # Test listing task tags with various exceptions
+    @patch("botocore.client.BaseClient._make_api_call", new=mock_make_api_call)
+    def test_list_task_tags_with_exceptions(self):
+        aws_provider = set_mocked_aws_provider()
+        datasync = DataSync(aws_provider)
+
+        tasks_by_name = {task.name: task for task in datasync.tasks.values()}
+        assert tasks_by_name["test_task"].tags == [
+            {"Key": "Name", "Value": "test_task"}
+        ]
+
+        # Tasks with exceptions should have empty tag lists
+        assert tasks_by_name["not_found_task"].tags == []
+        assert tasks_by_name["client_error_task"].tags == []
+        assert tasks_by_name["generic_error_task"].tags == []
@@ -0,0 +1,127 @@
+from unittest.mock import patch
+
+from tests.providers.aws.utils import AWS_REGION_US_EAST_1, set_mocked_aws_provider
+
+TASK_ID = "task-12345"
+TASK_ARN = f"arn:aws:datasync:{AWS_REGION_US_EAST_1}:123456789012:task/{TASK_ID}"
+
+
+class Test_datasync_task_logging_enabled:
+    def test_no_tasks(self):
+        from prowler.providers.aws.services.datasync.datasync_service import DataSync
+
+        # Set up a mocked AWS provider
+        mocked_aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
+
+        # Create a DataSync client with no tasks
+        datasync_client = DataSync(mocked_aws_provider)
+        datasync_client.tasks = {}
+
+        with patch(
+            "prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled.datasync_client",
+            new=datasync_client,
+        ), patch(
+            "prowler.providers.common.provider.Provider.get_global_provider",
+            return_value=mocked_aws_provider,
+        ):
+            from prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled import (
+                datasync_task_logging_enabled,
+            )
+
+            check = datasync_task_logging_enabled()
+            result = check.execute()
+            assert len(result) == 0
+
+    def test_task_without_logging(self):
+        from prowler.providers.aws.services.datasync.datasync_service import (
+            DataSync,
+            DataSyncTask,
+        )
+
+        # Set up a mocked AWS provider
+        mocked_aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
+
+        # Create a DataSync task without logging enabled
+        task = DataSyncTask(
+            id=TASK_ID,
+            arn=TASK_ARN,
+            name="TestTask",
+            region=AWS_REGION_US_EAST_1,
+            cloudwatch_log_group_arn=None,  # Logging not enabled
+            tags=[],
+        )
+
+        # Create a DataSync client with the task
+        datasync_client = DataSync(mocked_aws_provider)
+        datasync_client.tasks[TASK_ARN] = task
+
+        with patch(
+            "prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled.datasync_client",
+            new=datasync_client,
+        ), patch(
+            "prowler.providers.common.provider.Provider.get_global_provider",
+            return_value=mocked_aws_provider,
+        ):
+            from prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled import (
+                datasync_task_logging_enabled,
+            )
+
+            check = datasync_task_logging_enabled()
+            result = check.execute()
+            assert len(result) == 1
+            assert result[0].status == "FAIL"
+            assert (
+                result[0].status_extended
+                == f"DataSync task {task.name} does not have logging enabled."
+            )
+            assert result[0].resource_id == TASK_ID
+            assert result[0].resource_arn == TASK_ARN
+            assert result[0].region == AWS_REGION_US_EAST_1
+            assert result[0].resource_tags == []
+
+    def test_task_with_logging(self):
+        from prowler.providers.aws.services.datasync.datasync_service import (
+            DataSync,
+            DataSyncTask,
+        )
+
+        # Set up a mocked AWS provider
+        mocked_aws_provider = set_mocked_aws_provider([AWS_REGION_US_EAST_1])
+
+        # Create a DataSync task with logging enabled
+        task = DataSyncTask(
+            id=TASK_ID,
+            arn=TASK_ARN,
+            name="TestTask",
+            region=AWS_REGION_US_EAST_1,
+            cloudwatch_log_group_arn=f"arn:aws:logs:{AWS_REGION_US_EAST_1}:123456789012:log-group:datasync-log-group",
+            tags=[],
+        )
+
+        # Create a DataSync client with the task
+        datasync_client = DataSync(mocked_aws_provider)
+        datasync_client.tasks[TASK_ARN] = task
+
+        with patch(
+            "prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled.datasync_client",
+            new=datasync_client,
+        ), patch(
+            "prowler.providers.common.provider.Provider.get_global_provider",
+            return_value=mocked_aws_provider,
+        ):
+            from prowler.providers.aws.services.datasync.datasync_task_logging_enabled.datasync_task_logging_enabled import (
+                datasync_task_logging_enabled,
+            )
+
+            check = datasync_task_logging_enabled()
+            result = check.execute()
+            assert len(result) == 1
+            assert result[0].status == "PASS"
+            assert (
+                result[0].status_extended
+                == f"DataSync task {task.name} has logging enabled."
+            )
+            assert result[0].resource_id == TASK_ID
+            assert result[0].resource_arn == TASK_ARN
+            assert result[0].region == AWS_REGION_US_EAST_1
+            assert result[0].resource_tags == []