diff --git a/prowler/CHANGELOG.md b/prowler/CHANGELOG.md index eff2506c04..2268ad9d09 100644 --- a/prowler/CHANGELOG.md +++ b/prowler/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to the **Prowler SDK** are documented in this file. ## [5.24.1] (Prowler v5.24.1) +### 🚀 Added + +- `--repo-list-file` CLI flag for GitHub provider to load repositories from a file [(#10501)](https://github.com/prowler-cloud/prowler/pull/10501) + ### 🔄 Changed - `msgraph-sdk` from 1.23.0 to 1.55.0 and `azure-mgmt-resource` from 23.3.0 to 24.0.0, removing `marshmallow` as is a transitively dev dependency [(#10733)](https://github.com/prowler-cloud/prowler/pull/10733) diff --git a/prowler/providers/common/provider.py b/prowler/providers/common/provider.py index f2c77bdc79..610cd10f04 100644 --- a/prowler/providers/common/provider.py +++ b/prowler/providers/common/provider.py @@ -280,6 +280,7 @@ class Provider(ABC): mutelist_path=arguments.mutelist_file, config_path=arguments.config_file, repositories=repos, + repo_list_file=getattr(arguments, "repo_list_file", None), organizations=orgs, ) elif "googleworkspace" in provider_class_name.lower(): diff --git a/prowler/providers/github/exceptions/exceptions.py b/prowler/providers/github/exceptions/exceptions.py index b49cce8ebe..b0f472a76c 100644 --- a/prowler/providers/github/exceptions/exceptions.py +++ b/prowler/providers/github/exceptions/exceptions.py @@ -34,6 +34,14 @@ class GithubBaseException(ProwlerException): "message": "The provided provider ID does not match with the authenticated user or accessible organizations", "remediation": "Check the provider ID and ensure it matches the authenticated user or an organization you have access to.", }, + (5007, "GithubRepoListFileNotFoundError"): { + "message": "The repo list file was not found", + "remediation": "Check the file path and ensure it exists.", + }, + (5008, "GithubRepoListFileReadError"): { + "message": "Error reading the repo list file", + "remediation": "Check the file permissions and format.", + }, } def __init__(self, code, file=None, original_exception=None, message=None): @@ -104,3 +112,21 @@ class GithubInvalidProviderIdError(GithubCredentialsError): super().__init__( 5006, file=file, original_exception=original_exception, message=message ) + + +class GithubRepoListFileNotFoundError(GithubBaseException): + """Exception raised when the repo list file is not found.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 5007, file=file, original_exception=original_exception, message=message + ) + + +class GithubRepoListFileReadError(GithubBaseException): + """Exception raised when the repo list file cannot be read.""" + + def __init__(self, file=None, original_exception=None, message=None): + super().__init__( + 5008, file=file, original_exception=original_exception, message=message + ) diff --git a/prowler/providers/github/github_provider.py b/prowler/providers/github/github_provider.py index 16d13f7434..ab3441b81c 100644 --- a/prowler/providers/github/github_provider.py +++ b/prowler/providers/github/github_provider.py @@ -22,6 +22,8 @@ from prowler.providers.github.exceptions.exceptions import ( GithubInvalidCredentialsError, GithubInvalidProviderIdError, GithubInvalidTokenError, + GithubRepoListFileNotFoundError, + GithubRepoListFileReadError, GithubSetUpIdentityError, GithubSetUpSessionError, ) @@ -90,6 +92,8 @@ class GithubProvider(Provider): _type: str = "github" _auth_method: str = None + MAX_REPO_LIST_LINES: int = 10_000 + MAX_REPO_NAME_LENGTH: int = 500 _session: GithubSession _identity: GithubIdentityInfo _audit_config: dict @@ -113,6 +117,7 @@ class GithubProvider(Provider): mutelist_path: str = None, mutelist_content: dict = None, repositories: list = None, + repo_list_file: str = None, organizations: list = None, ): """ @@ -130,6 +135,7 @@ class GithubProvider(Provider): mutelist_path (str): Path to the mutelist file. mutelist_content (dict): Mutelist content. repositories (list): List of repository names to scan in 'owner/repo-name' format. + repo_list_file (str): Path to a file containing repository names (one per line). organizations (list): List of organization or user names to scan repositories for. """ logger.info("Instantiating GitHub Provider...") @@ -147,6 +153,10 @@ class GithubProvider(Provider): else: self._repositories = list(repositories) + # Load repos from file if provided + if repo_list_file: + self._load_repos_from_file(repo_list_file) + if organizations is None: self._organizations = [] elif isinstance(organizations, str): @@ -256,6 +266,46 @@ class GithubProvider(Provider): """ return self._organizations + def _load_repos_from_file(self, file_path: str) -> None: + """Load repository names from a file (one per line).""" + try: + repo_count = 0 + before = len(self._repositories) + with open(file_path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + repo_count += 1 + if repo_count > self.MAX_REPO_LIST_LINES: + raise GithubRepoListFileReadError( + file=file_path, + message=f"Repo list file exceeds maximum of {self.MAX_REPO_LIST_LINES} lines.", + ) + if len(line) > self.MAX_REPO_NAME_LENGTH: + logger.warning( + f"Skipping repo name exceeding {self.MAX_REPO_NAME_LENGTH} chars at line {repo_count} in {file_path}" + ) + continue + self._repositories.append(line) + self._repositories = list(dict.fromkeys(self._repositories)) + logger.info( + f"Loaded {len(self._repositories) - before} repositories from {file_path}" + ) + except FileNotFoundError: + raise GithubRepoListFileNotFoundError( + file=file_path, + message=f"Repo list file not found: {file_path}", + ) + except (GithubRepoListFileReadError, GithubRepoListFileNotFoundError): + raise + except Exception as error: + raise GithubRepoListFileReadError( + file=file_path, + original_exception=error, + message=f"Error reading repo list file: {error}", + ) + @staticmethod def setup_session( personal_access_token: str = None, diff --git a/prowler/providers/github/lib/arguments/arguments.py b/prowler/providers/github/lib/arguments/arguments.py index 748d77a927..946029ab43 100644 --- a/prowler/providers/github/lib/arguments/arguments.py +++ b/prowler/providers/github/lib/arguments/arguments.py @@ -50,6 +50,12 @@ def init_parser(self): default=None, metavar="REPOSITORY", ) + github_scoping_subparser.add_argument( + "--repo-list-file", + dest="repo_list_file", + default=None, + help="Path to a file containing a list of repositories to scan (one per line in 'owner/repo-name' format). Lines starting with # are treated as comments.", + ) github_scoping_subparser.add_argument( "--organization", "--organizations", diff --git a/tests/providers/github/github_provider_test.py b/tests/providers/github/github_provider_test.py index a87a35e038..00a3c33cbc 100644 --- a/tests/providers/github/github_provider_test.py +++ b/tests/providers/github/github_provider_test.py @@ -12,6 +12,8 @@ from prowler.providers.github.exceptions.exceptions import ( GithubInvalidCredentialsError, GithubInvalidProviderIdError, GithubInvalidTokenError, + GithubRepoListFileNotFoundError, + GithubRepoListFileReadError, GithubSetUpIdentityError, GithubSetUpSessionError, ) @@ -708,3 +710,81 @@ class Test_GithubProvider_Scoping: assert provider_none.repositories == [] assert provider_none.organizations == [] + + +class TestGitHubProviderLoadReposFromFile: + """Tests for GithubProvider._load_repos_from_file""" + + def _make_provider(self): + """Create a GithubProvider instance with mocked session/identity.""" + with ( + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_session", + return_value=GithubSession(token=PAT_TOKEN, id="", key=""), + ), + patch( + "prowler.providers.github.github_provider.GithubProvider.setup_identity", + return_value=GithubIdentityInfo( + account_id=ACCOUNT_ID, + account_name=ACCOUNT_NAME, + account_url=ACCOUNT_URL, + ), + ), + ): + provider = GithubProvider( + personal_access_token=PAT_TOKEN, + ) + return provider + + def test_load_repos_from_file_happy_path(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + repo_file.write_text("owner/repo-a\nowner/repo-b\nowner/repo-c\n") + + provider._load_repos_from_file(str(repo_file)) + + assert "owner/repo-a" in provider.repositories + assert "owner/repo-b" in provider.repositories + assert "owner/repo-c" in provider.repositories + + def test_load_repos_from_file_comments_and_blanks(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + repo_file.write_text( + "# This is a comment\n" + "\n" + "owner/repo-a\n" + " # Another comment\n" + " \n" + "owner/repo-b\n" + ) + + provider._load_repos_from_file(str(repo_file)) + + assert provider.repositories == ["owner/repo-a", "owner/repo-b"] + + def test_load_repos_from_file_not_found(self): + provider = self._make_provider() + + with pytest.raises(GithubRepoListFileNotFoundError): + provider._load_repos_from_file("/nonexistent/path/repos.txt") + + def test_load_repos_from_file_exceeds_max_lines(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + # Write MAX_REPO_LIST_LINES + 1 lines to trigger the guard + lines = [f"owner/repo-{i}" for i in range(provider.MAX_REPO_LIST_LINES + 1)] + repo_file.write_text("\n".join(lines) + "\n") + + with pytest.raises(GithubRepoListFileReadError): + provider._load_repos_from_file(str(repo_file)) + + def test_load_repos_from_file_skips_long_names(self, tmp_path): + provider = self._make_provider() + repo_file = tmp_path / "repos.txt" + long_name = "a" * (provider.MAX_REPO_NAME_LENGTH + 1) + repo_file.write_text(f"owner/valid-repo\n{long_name}\nowner/also-valid\n") + + provider._load_repos_from_file(str(repo_file)) + + assert provider.repositories == ["owner/valid-repo", "owner/also-valid"] diff --git a/tests/providers/github/lib/arguments/github_arguments_test.py b/tests/providers/github/lib/arguments/github_arguments_test.py index fd2f813b6e..20fd3f18be 100644 --- a/tests/providers/github/lib/arguments/github_arguments_test.py +++ b/tests/providers/github/lib/arguments/github_arguments_test.py @@ -82,13 +82,14 @@ class Test_GitHubArguments: arguments.init_parser(mock_github_args) # Verify scoping arguments were added - assert self.mock_scoping_group.add_argument.call_count == 2 + assert self.mock_scoping_group.add_argument.call_count == 3 # Check that all scoping arguments are present calls = self.mock_scoping_group.add_argument.call_args_list scoping_args = [call[0][0] for call in calls] assert "--repository" in scoping_args + assert "--repo-list-file" in scoping_args assert "--organization" in scoping_args def test_repository_argument_configuration(self): @@ -277,6 +278,33 @@ class Test_GitHubArguments_Integration: assert args.repository == ["owner1/repo1"] assert args.organization == ["org1"] + def test_real_argument_parsing_with_repo_list_file(self): + """Test parsing arguments with repo-list-file scoping""" + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + common_parser = argparse.ArgumentParser(add_help=False) + + mock_github_args = MagicMock() + mock_github_args.subparsers = subparsers + mock_github_args.common_providers_parser = common_parser + + arguments.init_parser(mock_github_args) + + # Parse arguments with repo-list-file + args = parser.parse_args( + [ + "github", + "--personal-access-token", + "test-token", + "--repo-list-file", + "/path/to/repos.txt", + ] + ) + + assert args.personal_access_token == "test-token" + assert args.repo_list_file == "/path/to/repos.txt" + assert args.repository is None + def test_real_argument_parsing_empty_scoping(self): """Test parsing arguments with empty scoping values""" parser = argparse.ArgumentParser()