ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-03-22 03:08:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): gracefully skip E2E when test directories are empty (#10311)

Browse Source
This commit is contained in:
Alan Buscaglia
2026-03-12 10:38:51 +01:00
committed by GitHub
parent fc2fef755a
commit e0d61ba5d1
3 changed files with 681 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

350
.github/scripts/test-e2e-path-resolution.sh vendored Executable file
View File

@@ -0,0 +1,350 @@
#!/usr/bin/env bash
#
# Test script for E2E test path resolution logic from ui-e2e-tests-v2.yml.
# Validates that the shell logic correctly transforms E2E_TEST_PATHS into
# Playwright-compatible paths.
#
# Usage: .github/scripts/test-e2e-path-resolution.sh
set -euo pipefail
# -- Colors ------------------------------------------------------------------
RED='\033[0;31m'
GREEN='\033[0;32m'
BOLD='\033[1m'
RESET='\033[0m'
# -- Counters ----------------------------------------------------------------
TOTAL=0
PASSED=0
FAILED=0
# -- Temp directory setup & cleanup ------------------------------------------
TMPDIR_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMPDIR_ROOT"' EXIT
# ---------------------------------------------------------------------------
# create_test_tree DIR [SUBDIRS_WITH_TESTS...]
#
# Creates a fake ui/tests/ tree inside DIR.
# All standard subdirs are created (empty).
# For each name in SUBDIRS_WITH_TESTS, a fake .spec.ts file is placed inside.
# ---------------------------------------------------------------------------
create_test_tree() {
local base="$1"; shift
local all_subdirs=(
auth home invitations profile providers scans
setups sign-in-base sign-up attack-paths findings
compliance browse manage-groups roles users overview
integrations
)
for d in "${all_subdirs[@]}"; do
mkdir -p "${base}/tests/${d}"
done
# Populate requested subdirs with a fake test file
for d in "$@"; do
mkdir -p "${base}/tests/${d}"
touch "${base}/tests/${d}/example.spec.ts"
done
}
# ---------------------------------------------------------------------------
# resolve_paths E2E_TEST_PATHS WORKING_DIR
#
# Extracted EXACT logic from .github/workflows/ui-e2e-tests-v2.yml lines 212-250.
# Outputs space-separated TEST_PATHS, or "SKIP" if no tests found.
# Must be run with WORKING_DIR as the cwd equivalent (we cd into it).
# ---------------------------------------------------------------------------
resolve_paths() {
local E2E_TEST_PATHS="$1"
local WORKING_DIR="$2"
(
cd "$WORKING_DIR"
# --- Line 212-214: strip ui/ prefix, strip **, deduplicate ---------------
TEST_PATHS="${E2E_TEST_PATHS}"
TEST_PATHS=$(echo "$TEST_PATHS" | sed 's|ui/||g' | sed 's|\*\*||g' | tr ' ' '\n' | sort -u)
# --- Line 216: drop setup helpers ----------------------------------------
TEST_PATHS=$(echo "$TEST_PATHS" | grep -v '^tests/setups/' || true)
# --- Lines 219-230: safety net for bare tests/ --------------------------
if echo "$TEST_PATHS" | grep -qx 'tests/'; then
SPECIFIC_DIRS=""
for dir in tests/*/; do
[[ "$dir" == "tests/setups/" ]] && continue
SPECIFIC_DIRS="${SPECIFIC_DIRS}${dir}"$'\n'
done
TEST_PATHS=$(echo "$TEST_PATHS" | grep -vx 'tests/' || true)
TEST_PATHS="${TEST_PATHS}"$'\n'"${SPECIFIC_DIRS}"
TEST_PATHS=$(echo "$TEST_PATHS" | grep -v '^$' | sort -u)
fi
# --- Lines 231-234: bail if empty ----------------------------------------
if [[ -z "$TEST_PATHS" ]]; then
echo "SKIP"
return
fi
# --- Lines 236-245: filter dirs with no test files -----------------------
VALID_PATHS=""
while IFS= read -r p; do
[[ -z "$p" ]] && continue
if find "$p" -name '*.spec.ts' -o -name '*.test.ts' 2>/dev/null | head -1 | grep -q .; then
VALID_PATHS="${VALID_PATHS}${p}"$'\n'
fi
done <<< "$TEST_PATHS"
VALID_PATHS=$(echo "$VALID_PATHS" | grep -v '^$')
# --- Lines 246-249: bail if all empty ------------------------------------
if [[ -z "$VALID_PATHS" ]]; then
echo "SKIP"
return
fi
# --- Line 250: final output (space-separated) ---------------------------
echo "$VALID_PATHS" | tr '\n' ' ' | sed 's/ $//'
)
}
# ---------------------------------------------------------------------------
# run_test NAME INPUT EXPECTED_TYPE [EXPECTED_VALUE]
#
# EXPECTED_TYPE is one of:
# "contains <path>" — output must contain this path
# "equals <value>" — output must exactly equal this value
# "skip" — expect SKIP (no runnable tests)
# "not_contains <p>" — output must NOT contain this path
#
# Multiple expectations can be specified by calling assert_* after run_test.
# For convenience, run_test supports a single assertion inline.
# ---------------------------------------------------------------------------
CURRENT_RESULT=""
CURRENT_TEST_NAME=""
run_test() {
local name="$1"
local input="$2"
local expect_type="$3"
local expect_value="${4:-}"
TOTAL=$((TOTAL + 1))
CURRENT_TEST_NAME="$name"
# Create a fresh temp tree per test
local test_dir="${TMPDIR_ROOT}/test_${TOTAL}"
mkdir -p "$test_dir"
# Default populated dirs: scans, providers, auth, home, profile, sign-up, sign-in-base
create_test_tree "$test_dir" scans providers auth home profile sign-up sign-in-base
CURRENT_RESULT=$(resolve_paths "$input" "$test_dir")
_check "$expect_type" "$expect_value"
}
# Like run_test but lets caller specify which subdirs have test files.
run_test_custom_tree() {
local name="$1"
local input="$2"
local expect_type="$3"
local expect_value="${4:-}"
shift 4
local populated_dirs=("$@")
TOTAL=$((TOTAL + 1))
CURRENT_TEST_NAME="$name"
local test_dir="${TMPDIR_ROOT}/test_${TOTAL}"
mkdir -p "$test_dir"
create_test_tree "$test_dir" "${populated_dirs[@]}"
CURRENT_RESULT=$(resolve_paths "$input" "$test_dir")
_check "$expect_type" "$expect_value"
}
_check() {
local expect_type="$1"
local expect_value="$2"
case "$expect_type" in
skip)
if [[ "$CURRENT_RESULT" == "SKIP" ]]; then
_pass
else
_fail "expected SKIP, got: '$CURRENT_RESULT'"
fi
;;
contains)
if [[ "$CURRENT_RESULT" == *"$expect_value"* ]]; then
_pass
else
_fail "expected to contain '$expect_value', got: '$CURRENT_RESULT'"
fi
;;
not_contains)
if [[ "$CURRENT_RESULT" != *"$expect_value"* ]]; then
_pass
else
_fail "expected NOT to contain '$expect_value', got: '$CURRENT_RESULT'"
fi
;;
equals)
if [[ "$CURRENT_RESULT" == "$expect_value" ]]; then
_pass
else
_fail "expected exactly '$expect_value', got: '$CURRENT_RESULT'"
fi
;;
*)
_fail "unknown expect_type: $expect_type"
;;
esac
}
_pass() {
PASSED=$((PASSED + 1))
printf '%b PASS%b %s\n' "$GREEN" "$RESET" "$CURRENT_TEST_NAME"
}
_fail() {
FAILED=$((FAILED + 1))
printf '%b FAIL%b %s\n' "$RED" "$RESET" "$CURRENT_TEST_NAME"
printf " %s\n" "$1"
}
# ===========================================================================
# TEST CASES
# ===========================================================================
echo ""
printf '%bE2E Path Resolution Tests%b\n' "$BOLD" "$RESET"
echo "=========================================="
# 1. Normal single module
run_test \
"1. Normal single module" \
"ui/tests/scans/**" \
"contains" "tests/scans/"
# 2. Multiple modules
run_test \
"2. Multiple modules — scans present" \
"ui/tests/scans/** ui/tests/providers/**" \
"contains" "tests/scans/"
run_test \
"2. Multiple modules — providers present" \
"ui/tests/scans/** ui/tests/providers/**" \
"contains" "tests/providers/"
# 3. Broad pattern (many modules)
run_test \
"3. Broad pattern — no bare tests/" \
"ui/tests/auth/** ui/tests/scans/** ui/tests/providers/** ui/tests/home/** ui/tests/profile/**" \
"not_contains" "tests/ "
# 4. Empty directory
run_test \
"4. Empty directory — skipped" \
"ui/tests/attack-paths/**" \
"skip"
# 5. Mix of populated and empty dirs
run_test \
"5. Mix populated+empty — scans present" \
"ui/tests/scans/** ui/tests/attack-paths/**" \
"contains" "tests/scans/"
run_test \
"5. Mix populated+empty — attack-paths absent" \
"ui/tests/scans/** ui/tests/attack-paths/**" \
"not_contains" "tests/attack-paths/"
# 6. All empty directories
run_test \
"6. All empty directories" \
"ui/tests/attack-paths/** ui/tests/findings/**" \
"skip"
# 7. Setup paths filtered
run_test \
"7. Setup paths filtered out" \
"ui/tests/setups/**" \
"skip"
# 8. Bare tests/ from broad pattern — safety net expands
run_test \
"8. Bare tests/ expands — scans present" \
"ui/tests/**" \
"contains" "tests/scans/"
run_test \
"8. Bare tests/ expands — setups excluded" \
"ui/tests/**" \
"not_contains" "tests/setups/"
# 9. Bare tests/ with all empty subdirs (only setups has files)
run_test_custom_tree \
"9. Bare tests/ — only setups has files" \
"ui/tests/**" \
"skip" "" \
setups
# 10. Duplicate paths
run_test \
"10. Duplicate paths — deduplicated" \
"ui/tests/scans/** ui/tests/scans/**" \
"equals" "tests/scans/"
# 11. Empty input
TOTAL=$((TOTAL + 1))
CURRENT_TEST_NAME="11. Empty input"
test_dir="${TMPDIR_ROOT}/test_${TOTAL}"
mkdir -p "$test_dir"
create_test_tree "$test_dir" scans providers
CURRENT_RESULT=$(resolve_paths "" "$test_dir")
_check "skip" ""
# 12. Trailing/leading whitespace
run_test \
"12. Whitespace handling" \
" ui/tests/scans/** " \
"contains" "tests/scans/"
# 13. Path without ui/ prefix
run_test \
"13. Path without ui/ prefix" \
"tests/scans/**" \
"contains" "tests/scans/"
# 14. Setup mixed with valid paths — only valid pass through
run_test \
"14. Setups + valid — setups filtered" \
"ui/tests/setups/** ui/tests/scans/**" \
"contains" "tests/scans/"
run_test \
"14. Setups + valid — setups absent" \
"ui/tests/setups/** ui/tests/scans/**" \
"not_contains" "tests/setups/"
# ===========================================================================
# SUMMARY
# ===========================================================================
echo ""
echo "=========================================="
if [[ "$FAILED" -eq 0 ]]; then
printf '%b%bAll tests passed: %d/%d%b\n' "$GREEN" "$BOLD" "$PASSED" "$TOTAL" "$RESET"
else
printf '%b%b%d/%d passed, %d FAILED%b\n' "$RED" "$BOLD" "$PASSED" "$TOTAL" "$FAILED" "$RESET"
fi
echo ""
exit "$FAILED"

17
.github/workflows/ui-e2e-tests-v2.yml vendored
View File

@@ -232,7 +232,22 @@ jobs:
echo "No runnable E2E test paths after filtering setups"
exit 0
fi
TEST_PATHS=$(echo "$TEST_PATHS" | tr '\n' ' ')
# Filter out directories that don't contain any test files
VALID_PATHS=""
while IFS= read -r p; do
[[ -z "$p" ]] && continue
if find "$p" -name '*.spec.ts' -o -name '*.test.ts' 2>/dev/null | head -1 | grep -q .; then
VALID_PATHS="${VALID_PATHS}${p}"$'\n'
else
echo "Skipping empty test directory: $p"
fi
done <<< "$TEST_PATHS"
VALID_PATHS=$(echo "$VALID_PATHS" | grep -v '^$')
if [[ -z "$VALID_PATHS" ]]; then
echo "No test files found in any resolved paths — skipping E2E"
exit 0
fi
TEST_PATHS=$(echo "$VALID_PATHS" | tr '\n' ' ')
echo "Resolved test paths: $TEST_PATHS"
pnpm exec playwright test $TEST_PATHS
fi

315
docs/developer-guide/test-impact-analysis.mdx Normal file
View File

@@ -0,0 +1,315 @@
---
title: 'Test Impact Analysis'
---
Test impact analysis (TIA) determines which tests to run based on the files changed in a pull request. Instead of running the full test suite on every pull request, TIA maps changed files to the specific Prowler SDK, API, and end-to-end (E2E) tests that cover them. This approach reduces continuous integration (CI) time and resource usage while maintaining confidence that relevant code paths are tested.
## Architecture
### Components
| Component | Path | Role |
|-----------|------|------|
| Configuration | `.github/test-impact.yml` | Defines ignored, critical, and module path mappings |
| Analysis engine | `.github/scripts/test-impact.py` | Python script that evaluates changed files against the configuration |
| Reusable workflow | `.github/workflows/test-impact-analysis.yml` | GitHub Actions reusable workflow that orchestrates the analysis |
| E2E consumer | `.github/workflows/ui-e2e-tests-v2.yml` | Consumes TIA outputs to run targeted Playwright tests |
### Flow Diagram
```
PR opened/updated
|
v
+-------------------------------+
| tj-actions/changed-files | Gets list of changed files from PR
+-------------------------------+
|
v
+-------------------------------+
| test-impact.py |
| |
| 1. Filter ignored paths | docs/**, *.md, .gitignore, etc.
| 2. Check critical paths | prowler/lib/**, ui/lib/**, .github/workflows/**
| 3. Match modules | Map remaining files to module definitions
| 4. Categorize tests | Split into sdk-tests, api-tests, ui-e2e
+-------------------------------+
|
v
+-------------------------------+
| GitHub Actions Outputs |
| |
| run-all: true/false |
| sdk-tests: "tests/providers/aws/**"
| api-tests: "api/src/backend/api/tests/**"
| ui-e2e: "ui/tests/providers/**"
| modules: "sdk-aws,ui-providers"
| has-tests: true/false |
| has-sdk-tests: true/false |
| has-api-tests: true/false |
| has-ui-e2e: true/false |
+-------------------------------+
|
v
+-------------------------------+
| Consumer Workflows |
| |
| ui-e2e-tests-v2.yml: |
| - Path resolution pipeline |
| - Playwright execution |
+-------------------------------+
```
## Configuration Reference
The configuration lives in `.github/test-impact.yml` and contains three sections.
### `ignored` — Paths That Never Trigger Tests
Files matching these patterns are filtered out before any analysis takes place. This section is intended for non-code files.
```yaml
ignored:
paths:
- docs/**
- "*.md"
- .gitignore
- skills/**
- ui/tests/setups/** # E2E auth setup helpers (not runnable tests)
```
### `critical` — Paths That Trigger All Tests
If any changed file matches a critical path, the system short-circuits and outputs `run-all: true`. All downstream consumers then run their complete test suites.
```yaml
critical:
paths:
- prowler/lib/** # SDK core
- ui/lib/** # UI shared utilities
- ui/playwright.config.ts # Test infrastructure
- .github/workflows/** # CI changes
- .github/test-impact.yml # This config itself
```
### `modules` — Path-to-Test Mappings
Each module maps source file patterns to the tests that cover them.
```yaml
- name: ui-providers # Unique identifier
match: # Source file glob patterns
- ui/components/providers/**
- ui/actions/providers/**
- ui/app/**/providers/**
- ui/tests/providers/** # Test file changes also trigger themselves
tests: [] # SDK/API unit test patterns (empty for UI modules)
e2e: # Playwright E2E test patterns
- ui/tests/providers/**
```
#### Module Schema
| Field | Type | Description |
|-------|------|-------------|
| `name` | `string` | Unique module identifier (for example, `sdk-aws`, `ui-providers`, `api-views`) |
| `match` | `list[glob]` | Source file patterns that trigger this module |
| `tests` | `list[glob]` | Prowler SDK (`tests/`) or API (`api/`) unit test patterns to run |
| `e2e` | `list[glob]` | UI E2E test patterns (`ui/tests/`) to run |
#### Module Categories
- **`sdk-*`:** Provider and lib modules. These only produce `tests` output, not `e2e`.
- **`api-*`:** API views, serializers, filters, and role-based access control (RBAC). These produce `tests` and sometimes `e2e` (API changes can affect UI flows).
- **`ui-*`:** UI feature modules. These only produce `e2e` output, not `tests`.
## Path Resolution Pipeline
The E2E consumer workflow (`.github/workflows/ui-e2e-tests-v2.yml`, lines 202–253) transforms the `ui-e2e` output from glob patterns into paths that Playwright can execute. This transformation follows a multi-step shell pipeline.
### Step 1: Check Run Mode
```bash
if [[ "${RUN_ALL_TESTS}" == "true" ]]; then
pnpm run test:e2e # Run everything, skip pipeline
fi
```
### Step 2: Strip the `ui/` Prefix and `**` Suffix
```bash
# "ui/tests/providers/**" -> "tests/providers/"
TEST_PATHS=$(echo "$E2E_TEST_PATHS" | sed 's|ui/||g' | sed 's|\*\*||g')
```
### Step 3: Filter Out Setup Paths
```bash
# Remove auth setup helpers (not runnable test suites)
TEST_PATHS=$(echo "$TEST_PATHS" | grep -v '^tests/setups/')
```
### Step 4: Safety Net for Bare `tests/`
If the pattern `ui/tests/**` was present in the output (from a critical path or a broad module like `ui-shadcn`), it resolves to bare `tests/` after stripping. This would cause Playwright to discover setup files in `tests/setups/`, so it gets expanded instead:
```bash
if echo "$TEST_PATHS" | grep -qx 'tests/'; then
# Expand to specific subdirs, excluding tests/setups/
for dir in tests/*/; do
[[ "$dir" == "tests/setups/" ]] && continue
SPECIFIC_DIRS="${SPECIFIC_DIRS}${dir}"
done
fi
```
### Step 5: Empty Directory Check
Directories that do not contain any `.spec.ts` or `.test.ts` files are skipped. This handles forward-looking patterns where a module is configured but tests have not been written yet.
```bash
if find "$p" -name '*.spec.ts' -o -name '*.test.ts' | head -1 | grep -q .; then
VALID_PATHS="${VALID_PATHS}${p}"
else
echo "Skipping empty test directory: $p"
fi
```
### Step 6: Execute Playwright
```bash
pnpm exec playwright test $TEST_PATHS
# For example: pnpm exec playwright test tests/providers/ tests/scans/
```
## Playwright Project Mapping
Playwright discovers tests by scanning the directories passed to it. The `playwright.config.ts` file defines projects with `testMatch` patterns that control which spec files each project claims:
```
tests/providers/providers.spec.ts -> "providers" project -> depends on admin.auth.setup
tests/scans/scans.spec.ts -> "scans" project -> depends on admin.auth.setup
tests/sign-in-base/*.spec.ts -> "sign-in-base" -> no auth dependency
tests/auth/*.spec.ts -> "auth" -> no auth dependency
tests/sign-up/sign-up.spec.ts -> "sign-up" -> no auth dependency
tests/invitations/invitations.spec.ts -> "invitations" -> depends on admin.auth.setup
```
Auth setup projects (`admin.auth.setup`, `manage-scans.auth.setup`, and others) create authenticated browser state files. Projects that declare them as `dependencies` wait for the setup to complete before running.
When TIA runs only `tests/providers/`, Playwright still automatically runs `admin.auth.setup` because the `providers` project declares it as a dependency.
## Edge Cases and Known Considerations
### Forward-Looking Patterns (Empty Test Directories)
A module can reference `ui/tests/attack-paths/**` before any tests exist there. The empty directory check (step 5) gracefully skips it instead of failing.
### Broad Patterns and the Safety Net
Modules like `ui-shadcn` and `api-views` list every E2E test suite explicitly to avoid using `ui/tests/**`. If a broad pattern does produce bare `tests/`, the safety net expands it to specific subdirectories, excluding `tests/setups/`.
### Setup Files and Auth Dependencies
`ui/tests/setups/**` is listed in the `ignored` section and also filtered in the path resolution pipeline. This double protection ensures setup files are never passed as test targets to Playwright. Auth setups run only when declared as project dependencies.
### Critical Path Triggering Run-All
Changes to `.github/workflows/**` or `.github/test-impact.yml` trigger `run-all: true`. This means editing any workflow file (even unrelated ones) runs the full test suite. This behavior is intentional — CI infrastructure changes should be validated broadly.
### Unmatched Files
Files that do not match any ignored, critical, or module pattern produce no test output. The `has-tests` flag is set to `false` and consumer workflows skip entirely via the `skip-e2e` job.
## Adding New Test Modules
To add tests for a new UI feature (for example, `dashboards`):
1. **Add the module to `.github/test-impact.yml`:**
```yaml
- name: ui-dashboards
match:
- ui/components/dashboards/**
- ui/actions/dashboards/**
- ui/app/**/dashboards/**
- ui/tests/dashboards/**
tests: []
e2e:
- ui/tests/dashboards/**
```
2. **Create the test directory and spec file:**
```
ui/tests/dashboards/dashboards.spec.ts
```
3. **Add a Playwright project in `ui/playwright.config.ts`:**
```typescript
{
name: "dashboards",
testMatch: "dashboards.spec.ts",
dependencies: ["admin.auth.setup"], // if tests need auth
},
```
4. **Register E2E paths in shared UI modules (if applicable):**
If the feature uses shared UI components, add the E2E path to the `ui-shadcn` module so that changes to shared components also trigger dashboard tests:
```yaml
- name: ui-shadcn
match:
- ui/components/shadcn/**
- ui/components/ui/**
e2e:
- ui/tests/dashboards/** # Add here
# ... existing paths
```
5. **Register E2E paths in API modules (if applicable):**
If API changes affect this feature, add the E2E path to the relevant `api-*` module (for example, `api-views`).
## Troubleshooting
### Tests Not Running When Expected
1. Check whether the changed file matches an `ignored` pattern. The script logs `[IGNORED]` to stderr.
2. Verify the file matches a module's `match` pattern. To test locally, run:
```bash
python .github/scripts/test-impact.py path/to/changed/file.ts
```
3. Confirm the module has non-empty `e2e` (for E2E) or `tests` (for unit tests).
4. Check the `has-ui-e2e` output — the consumer workflow gates on this flag.
### Unexpected Auth Setup Errors
Auth setup projects run automatically when a test project declares them as `dependencies`. If auth failures occur:
- **Verify secrets:** Confirm that the `E2E_ADMIN_USER` and `E2E_ADMIN_PASSWORD` secrets are set.
- **Check setup file existence:** Ensure the auth setup file exists in `ui/tests/setups/`.
- **Validate test match patterns:** Ensure the `testMatch` pattern in `playwright.config.ts` correctly matches the setup file.
### "No Tests Found" Errors
This typically means the path resolution pipeline produced valid directories but Playwright could not match any spec files to a project:
- **Check project configuration:** Verify that `playwright.config.ts` has a project with a `testMatch` pattern for the spec files in that directory.
- **Verify file naming:** Confirm the spec file naming matches the expected pattern (for example, `feature.spec.ts`).
### "No Runnable E2E Test Paths After Filtering Setups"
All resolved paths were under `tests/setups/`. This indicates the module's `e2e` patterns only point to setup files, which is a configuration error. The module should be updated to point to actual test directories.
### Debugging Locally
```bash
# See what the analysis engine produces for specific files
python .github/scripts/test-impact.py ui/components/providers/some-file.tsx
# Output goes to stderr (analysis log) and GITHUB_OUTPUT (structured output)
# Without the GITHUB_OUTPUT env var, results print to stderr only
```