feat(github): add --repo-list-file flag for GitHub scanning (#10501)

Co-authored-by: Raajhesh Kannaa Chidambaram <495042+raajheshkannaa@users.noreply.github.com>
Co-authored-by: Andoni A. <14891798+andoniaf@users.noreply.github.com>

prowler/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to the **Prowler SDK** are documented in this file.
 
 ## [5.24.1] (Prowler v5.24.1)
 
+### 🚀 Added
+
+- `--repo-list-file` CLI flag for GitHub provider to load repositories from a file [(#10501)](https://github.com/prowler-cloud/prowler/pull/10501)
+
 ### 🔄 Changed
 
 - `msgraph-sdk` from 1.23.0 to 1.55.0 and `azure-mgmt-resource` from 23.3.0 to 24.0.0, removing `marshmallow` as is a transitively dev dependency [(#10733)](https://github.com/prowler-cloud/prowler/pull/10733)

prowler/providers/common/provider.py

@@ -280,6 +280,7 @@ class Provider(ABC):
                         mutelist_path=arguments.mutelist_file,
                         config_path=arguments.config_file,
                         repositories=repos,
+                        repo_list_file=getattr(arguments, "repo_list_file", None),
                         organizations=orgs,
                     )
                 elif "googleworkspace" in provider_class_name.lower():

prowler/providers/github/exceptions/exceptions.py

@@ -34,6 +34,14 @@ class GithubBaseException(ProwlerException):
             "message": "The provided provider ID does not match with the authenticated user or accessible organizations",
             "remediation": "Check the provider ID and ensure it matches the authenticated user or an organization you have access to.",
         },
+        (5007, "GithubRepoListFileNotFoundError"): {
+            "message": "The repo list file was not found",
+            "remediation": "Check the file path and ensure it exists.",
+        },
+        (5008, "GithubRepoListFileReadError"): {
+            "message": "Error reading the repo list file",
+            "remediation": "Check the file permissions and format.",
+        },
     }
 
     def __init__(self, code, file=None, original_exception=None, message=None):
@@ -104,3 +112,21 @@ class GithubInvalidProviderIdError(GithubCredentialsError):
         super().__init__(
             5006, file=file, original_exception=original_exception, message=message
         )
+
+
+class GithubRepoListFileNotFoundError(GithubBaseException):
+    """Exception raised when the repo list file is not found."""
+
+    def __init__(self, file=None, original_exception=None, message=None):
+        super().__init__(
+            5007, file=file, original_exception=original_exception, message=message
+        )
+
+
+class GithubRepoListFileReadError(GithubBaseException):
+    """Exception raised when the repo list file cannot be read."""
+
+    def __init__(self, file=None, original_exception=None, message=None):
+        super().__init__(
+            5008, file=file, original_exception=original_exception, message=message
+        )

prowler/providers/github/github_provider.py

@@ -22,6 +22,8 @@ from prowler.providers.github.exceptions.exceptions import (
     GithubInvalidCredentialsError,
     GithubInvalidProviderIdError,
     GithubInvalidTokenError,
+    GithubRepoListFileNotFoundError,
+    GithubRepoListFileReadError,
     GithubSetUpIdentityError,
     GithubSetUpSessionError,
 )
@@ -90,6 +92,8 @@ class GithubProvider(Provider):
 
     _type: str = "github"
     _auth_method: str = None
+    MAX_REPO_LIST_LINES: int = 10_000
+    MAX_REPO_NAME_LENGTH: int = 500
     _session: GithubSession
     _identity: GithubIdentityInfo
     _audit_config: dict
@@ -113,6 +117,7 @@ class GithubProvider(Provider):
         mutelist_path: str = None,
         mutelist_content: dict = None,
         repositories: list = None,
+        repo_list_file: str = None,
         organizations: list = None,
     ):
         """
@@ -130,6 +135,7 @@ class GithubProvider(Provider):
             mutelist_path (str): Path to the mutelist file.
             mutelist_content (dict): Mutelist content.
             repositories (list): List of repository names to scan in 'owner/repo-name' format.
+            repo_list_file (str): Path to a file containing repository names (one per line).
             organizations (list): List of organization or user names to scan repositories for.
         """
         logger.info("Instantiating GitHub Provider...")
@@ -147,6 +153,10 @@ class GithubProvider(Provider):
         else:
             self._repositories = list(repositories)
 
+        # Load repos from file if provided
+        if repo_list_file:
+            self._load_repos_from_file(repo_list_file)
+
         if organizations is None:
             self._organizations = []
         elif isinstance(organizations, str):
@@ -256,6 +266,46 @@ class GithubProvider(Provider):
         """
         return self._organizations
 
+    def _load_repos_from_file(self, file_path: str) -> None:
+        """Load repository names from a file (one per line)."""
+        try:
+            repo_count = 0
+            before = len(self._repositories)
+            with open(file_path, "r") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line or line.startswith("#"):
+                        continue
+                    repo_count += 1
+                    if repo_count > self.MAX_REPO_LIST_LINES:
+                        raise GithubRepoListFileReadError(
+                            file=file_path,
+                            message=f"Repo list file exceeds maximum of {self.MAX_REPO_LIST_LINES} lines.",
+                        )
+                    if len(line) > self.MAX_REPO_NAME_LENGTH:
+                        logger.warning(
+                            f"Skipping repo name exceeding {self.MAX_REPO_NAME_LENGTH} chars at line {repo_count} in {file_path}"
+                        )
+                        continue
+                    self._repositories.append(line)
+            self._repositories = list(dict.fromkeys(self._repositories))
+            logger.info(
+                f"Loaded {len(self._repositories) - before} repositories from {file_path}"
+            )
+        except FileNotFoundError:
+            raise GithubRepoListFileNotFoundError(
+                file=file_path,
+                message=f"Repo list file not found: {file_path}",
+            )
+        except (GithubRepoListFileReadError, GithubRepoListFileNotFoundError):
+            raise
+        except Exception as error:
+            raise GithubRepoListFileReadError(
+                file=file_path,
+                original_exception=error,
+                message=f"Error reading repo list file: {error}",
+            )
+
     @staticmethod
     def setup_session(
         personal_access_token: str = None,

prowler/providers/github/lib/arguments/arguments.py

@@ -50,6 +50,12 @@ def init_parser(self):
         default=None,
         metavar="REPOSITORY",
     )
+    github_scoping_subparser.add_argument(
+        "--repo-list-file",
+        dest="repo_list_file",
+        default=None,
+        help="Path to a file containing a list of repositories to scan (one per line in 'owner/repo-name' format). Lines starting with # are treated as comments.",
+    )
     github_scoping_subparser.add_argument(
         "--organization",
         "--organizations",

tests/providers/github/github_provider_test.py

@@ -12,6 +12,8 @@ from prowler.providers.github.exceptions.exceptions import (
     GithubInvalidCredentialsError,
     GithubInvalidProviderIdError,
     GithubInvalidTokenError,
+    GithubRepoListFileNotFoundError,
+    GithubRepoListFileReadError,
     GithubSetUpIdentityError,
     GithubSetUpSessionError,
 )
@@ -708,3 +710,81 @@ class Test_GithubProvider_Scoping:
 
             assert provider_none.repositories == []
             assert provider_none.organizations == []
+
+
+class TestGitHubProviderLoadReposFromFile:
+    """Tests for GithubProvider._load_repos_from_file"""
+
+    def _make_provider(self):
+        """Create a GithubProvider instance with mocked session/identity."""
+        with (
+            patch(
+                "prowler.providers.github.github_provider.GithubProvider.setup_session",
+                return_value=GithubSession(token=PAT_TOKEN, id="", key=""),
+            ),
+            patch(
+                "prowler.providers.github.github_provider.GithubProvider.setup_identity",
+                return_value=GithubIdentityInfo(
+                    account_id=ACCOUNT_ID,
+                    account_name=ACCOUNT_NAME,
+                    account_url=ACCOUNT_URL,
+                ),
+            ),
+        ):
+            provider = GithubProvider(
+                personal_access_token=PAT_TOKEN,
+            )
+        return provider
+
+    def test_load_repos_from_file_happy_path(self, tmp_path):
+        provider = self._make_provider()
+        repo_file = tmp_path / "repos.txt"
+        repo_file.write_text("owner/repo-a\nowner/repo-b\nowner/repo-c\n")
+
+        provider._load_repos_from_file(str(repo_file))
+
+        assert "owner/repo-a" in provider.repositories
+        assert "owner/repo-b" in provider.repositories
+        assert "owner/repo-c" in provider.repositories
+
+    def test_load_repos_from_file_comments_and_blanks(self, tmp_path):
+        provider = self._make_provider()
+        repo_file = tmp_path / "repos.txt"
+        repo_file.write_text(
+            "# This is a comment\n"
+            "\n"
+            "owner/repo-a\n"
+            "  # Another comment\n"
+            "  \n"
+            "owner/repo-b\n"
+        )
+
+        provider._load_repos_from_file(str(repo_file))
+
+        assert provider.repositories == ["owner/repo-a", "owner/repo-b"]
+
+    def test_load_repos_from_file_not_found(self):
+        provider = self._make_provider()
+
+        with pytest.raises(GithubRepoListFileNotFoundError):
+            provider._load_repos_from_file("/nonexistent/path/repos.txt")
+
+    def test_load_repos_from_file_exceeds_max_lines(self, tmp_path):
+        provider = self._make_provider()
+        repo_file = tmp_path / "repos.txt"
+        # Write MAX_REPO_LIST_LINES + 1 lines to trigger the guard
+        lines = [f"owner/repo-{i}" for i in range(provider.MAX_REPO_LIST_LINES + 1)]
+        repo_file.write_text("\n".join(lines) + "\n")
+
+        with pytest.raises(GithubRepoListFileReadError):
+            provider._load_repos_from_file(str(repo_file))
+
+    def test_load_repos_from_file_skips_long_names(self, tmp_path):
+        provider = self._make_provider()
+        repo_file = tmp_path / "repos.txt"
+        long_name = "a" * (provider.MAX_REPO_NAME_LENGTH + 1)
+        repo_file.write_text(f"owner/valid-repo\n{long_name}\nowner/also-valid\n")
+
+        provider._load_repos_from_file(str(repo_file))
+
+        assert provider.repositories == ["owner/valid-repo", "owner/also-valid"]

tests/providers/github/lib/arguments/github_arguments_test.py

@@ -82,13 +82,14 @@ class Test_GitHubArguments:
         arguments.init_parser(mock_github_args)
 
         # Verify scoping arguments were added
-        assert self.mock_scoping_group.add_argument.call_count == 2
+        assert self.mock_scoping_group.add_argument.call_count == 3
 
         # Check that all scoping arguments are present
         calls = self.mock_scoping_group.add_argument.call_args_list
         scoping_args = [call[0][0] for call in calls]
 
         assert "--repository" in scoping_args
+        assert "--repo-list-file" in scoping_args
         assert "--organization" in scoping_args
 
     def test_repository_argument_configuration(self):
@@ -277,6 +278,33 @@ class Test_GitHubArguments_Integration:
         assert args.repository == ["owner1/repo1"]
         assert args.organization == ["org1"]
 
+    def test_real_argument_parsing_with_repo_list_file(self):
+        """Test parsing arguments with repo-list-file scoping"""
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers()
+        common_parser = argparse.ArgumentParser(add_help=False)
+
+        mock_github_args = MagicMock()
+        mock_github_args.subparsers = subparsers
+        mock_github_args.common_providers_parser = common_parser
+
+        arguments.init_parser(mock_github_args)
+
+        # Parse arguments with repo-list-file
+        args = parser.parse_args(
+            [
+                "github",
+                "--personal-access-token",
+                "test-token",
+                "--repo-list-file",
+                "/path/to/repos.txt",
+            ]
+        )
+
+        assert args.personal_access_token == "test-token"
+        assert args.repo_list_file == "/path/to/repos.txt"
+        assert args.repository is None
+
     def test_real_argument_parsing_empty_scoping(self):
         """Test parsing arguments with empty scoping values"""
         parser = argparse.ArgumentParser()