diff --git a/.github/test-impact.yml b/.github/test-impact.yml index 3f90ad3d3a..0626f33707 100644 --- a/.github/test-impact.yml +++ b/.github/test-impact.yml @@ -14,7 +14,7 @@ ignored: - "*.md" - "**/*.md" - mkdocs.yml - + # Config files that don't affect runtime - .gitignore - .gitattributes @@ -23,7 +23,7 @@ ignored: - .backportrc.json - CODEOWNERS - LICENSE - + # IDE/Editor configs - .vscode/** - .idea/** @@ -31,10 +31,13 @@ ignored: # Examples and contrib (not production code) - examples/** - contrib/** - + # Skills (AI agent configs, not runtime) - skills/** - + + # E2E setup helpers (not runnable tests) + - ui/tests/setups/** + # Permissions docs - permissions/** @@ -47,18 +50,18 @@ critical: - prowler/config/** - prowler/exceptions/** - prowler/providers/common/** - + # API Core - api/src/backend/api/models.py - api/src/backend/config/** - api/src/backend/conftest.py - + # UI Core - ui/lib/** - ui/types/** - ui/config/** - ui/middleware.ts - + # CI/CD changes - .github/workflows/** - .github/test-impact.yml diff --git a/.github/workflows/ui-e2e-tests-v2.yml b/.github/workflows/ui-e2e-tests-v2.yml index 8de1e9d9fa..caa712f3dd 100644 --- a/.github/workflows/ui-e2e-tests-v2.yml +++ b/.github/workflows/ui-e2e-tests-v2.yml @@ -25,7 +25,7 @@ jobs: e2e-tests: needs: impact-analysis if: | - github.repository == 'prowler-cloud/prowler' && + github.repository == 'prowler-cloud/prowler' && (needs.impact-analysis.outputs.has-ui-e2e == 'true' || needs.impact-analysis.outputs.run-all == 'true') runs-on: ubuntu-latest env: @@ -200,7 +200,14 @@ jobs: # e.g., "ui/tests/providers/**" -> "tests/providers" TEST_PATHS="${{ env.E2E_TEST_PATHS }}" # Remove ui/ prefix and convert ** to empty (playwright handles recursion) - TEST_PATHS=$(echo "$TEST_PATHS" | sed 's|ui/||g' | sed 's|\*\*||g' | tr ' ' '\n' | sort -u | tr '\n' ' ') + TEST_PATHS=$(echo "$TEST_PATHS" | sed 's|ui/||g' | sed 's|\*\*||g' | tr ' ' '\n' | sort -u) + # Drop auth setup helpers (not runnable test suites) + TEST_PATHS=$(echo "$TEST_PATHS" | grep -v '^tests/setups/') + if [[ -z "$TEST_PATHS" ]]; then + echo "No runnable E2E test paths after filtering setups" + exit 0 + fi + TEST_PATHS=$(echo "$TEST_PATHS" | tr '\n' ' ') echo "Resolved test paths: $TEST_PATHS" pnpm exec playwright test $TEST_PATHS fi @@ -222,8 +229,8 @@ jobs: skip-e2e: needs: impact-analysis if: | - github.repository == 'prowler-cloud/prowler' && - needs.impact-analysis.outputs.has-ui-e2e != 'true' && + github.repository == 'prowler-cloud/prowler' && + needs.impact-analysis.outputs.has-ui-e2e != 'true' && needs.impact-analysis.outputs.run-all != 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/ui-e2e-tests.yml b/.github/workflows/ui-e2e-tests.yml deleted file mode 100644 index 6dcc2f73af..0000000000 --- a/.github/workflows/ui-e2e-tests.yml +++ /dev/null @@ -1,172 +0,0 @@ -name: UI - E2E Tests - -on: - pull_request: - branches: - - master - - "v5.*" - paths: - - '.github/workflows/ui-e2e-tests.yml' - - 'ui/**' - -jobs: - - e2e-tests: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - env: - AUTH_SECRET: 'fallback-ci-secret-for-testing' - AUTH_TRUST_HOST: true - NEXTAUTH_URL: 'http://localhost:3000' - NEXT_PUBLIC_API_BASE_URL: 'http://localhost:8080/api/v1' - E2E_ADMIN_USER: ${{ secrets.E2E_ADMIN_USER }} - E2E_ADMIN_PASSWORD: ${{ secrets.E2E_ADMIN_PASSWORD }} - E2E_AWS_PROVIDER_ACCOUNT_ID: ${{ secrets.E2E_AWS_PROVIDER_ACCOUNT_ID }} - E2E_AWS_PROVIDER_ACCESS_KEY: ${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }} - E2E_AWS_PROVIDER_SECRET_KEY: ${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }} - E2E_AWS_PROVIDER_ROLE_ARN: ${{ secrets.E2E_AWS_PROVIDER_ROLE_ARN }} - E2E_AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }} - E2E_AZURE_CLIENT_ID: ${{ secrets.E2E_AZURE_CLIENT_ID }} - E2E_AZURE_SECRET_ID: ${{ secrets.E2E_AZURE_SECRET_ID }} - E2E_AZURE_TENANT_ID: ${{ secrets.E2E_AZURE_TENANT_ID }} - E2E_M365_DOMAIN_ID: ${{ secrets.E2E_M365_DOMAIN_ID }} - E2E_M365_CLIENT_ID: ${{ secrets.E2E_M365_CLIENT_ID }} - E2E_M365_SECRET_ID: ${{ secrets.E2E_M365_SECRET_ID }} - E2E_M365_TENANT_ID: ${{ secrets.E2E_M365_TENANT_ID }} - E2E_M365_CERTIFICATE_CONTENT: ${{ secrets.E2E_M365_CERTIFICATE_CONTENT }} - E2E_KUBERNETES_CONTEXT: 'kind-kind' - E2E_KUBERNETES_KUBECONFIG_PATH: /home/runner/.kube/config - E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY: ${{ secrets.E2E_GCP_BASE64_SERVICE_ACCOUNT_KEY }} - E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }} - E2E_GITHUB_APP_ID: ${{ secrets.E2E_GITHUB_APP_ID }} - E2E_GITHUB_BASE64_APP_PRIVATE_KEY: ${{ secrets.E2E_GITHUB_BASE64_APP_PRIVATE_KEY }} - E2E_GITHUB_USERNAME: ${{ secrets.E2E_GITHUB_USERNAME }} - E2E_GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.E2E_GITHUB_PERSONAL_ACCESS_TOKEN }} - E2E_GITHUB_ORGANIZATION: ${{ secrets.E2E_GITHUB_ORGANIZATION }} - E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN: ${{ secrets.E2E_GITHUB_ORGANIZATION_ACCESS_TOKEN }} - E2E_ORGANIZATION_ID: ${{ secrets.E2E_ORGANIZATION_ID }} - E2E_OCI_TENANCY_ID: ${{ secrets.E2E_OCI_TENANCY_ID }} - E2E_OCI_USER_ID: ${{ secrets.E2E_OCI_USER_ID }} - E2E_OCI_FINGERPRINT: ${{ secrets.E2E_OCI_FINGERPRINT }} - E2E_OCI_KEY_CONTENT: ${{ secrets.E2E_OCI_KEY_CONTENT }} - E2E_OCI_REGION: ${{ secrets.E2E_OCI_REGION }} - E2E_NEW_USER_PASSWORD: ${{ secrets.E2E_NEW_USER_PASSWORD }} - - steps: - - name: Checkout repository - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 - with: - cluster_name: kind - - name: Modify kubeconfig - run: | - # Modify the kubeconfig to use the kind cluster server to https://kind-control-plane:6443 - # from worker service into docker-compose.yml - kubectl config set-cluster kind-kind --server=https://kind-control-plane:6443 - kubectl config view - - name: Add network kind to docker compose - run: | - # Add the network kind to the docker compose to interconnect to kind cluster - yq -i '.networks.kind.external = true' docker-compose.yml - # Add network kind to worker service and default network too - yq -i '.services.worker.networks = ["kind","default"]' docker-compose.yml - - name: Fix API data directory permissions - run: docker run --rm -v $(pwd)/_data/api:/data alpine chown -R 1000:1000 /data - - name: Add AWS credentials for testing AWS SDK Default Adding Provider - run: | - echo "Adding AWS credentials for testing AWS SDK Default Adding Provider..." - echo "AWS_ACCESS_KEY_ID=${{ secrets.E2E_AWS_PROVIDER_ACCESS_KEY }}" >> .env - echo "AWS_SECRET_ACCESS_KEY=${{ secrets.E2E_AWS_PROVIDER_SECRET_KEY }}" >> .env - - name: Start API services - run: | - # Override docker-compose image tag to use latest instead of stable - # This overrides any PROWLER_API_VERSION set in .env file - export PROWLER_API_VERSION=latest - echo "Using PROWLER_API_VERSION=${PROWLER_API_VERSION}" - docker compose up -d api worker worker-beat - - name: Wait for API to be ready - run: | - echo "Waiting for prowler-api..." - timeout=150 # 5 minutes max - elapsed=0 - while [ $elapsed -lt $timeout ]; do - if curl -s ${NEXT_PUBLIC_API_BASE_URL}/docs >/dev/null 2>&1; then - echo "Prowler API is ready!" - exit 0 - fi - echo "Waiting for prowler-api... (${elapsed}s elapsed)" - sleep 5 - elapsed=$((elapsed + 5)) - done - echo "Timeout waiting for prowler-api to start" - exit 1 - - name: Load database fixtures for E2E tests - run: | - docker compose exec -T api sh -c ' - echo "Loading all fixtures from api/fixtures/dev/..." - for fixture in api/fixtures/dev/*.json; do - if [ -f "$fixture" ]; then - echo "Loading $fixture" - poetry run python manage.py loaddata "$fixture" --database admin - fi - done - echo "All database fixtures loaded successfully!" - ' - - name: Setup Node.js environment - uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 - with: - node-version: '24.13.0' - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 10 - run_install: false - - name: Get pnpm store directory - shell: bash - run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - name: Setup pnpm and Next.js cache - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 - with: - path: | - ${{ env.STORE_PATH }} - ./ui/node_modules - ./ui/.next/cache - key: ${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}-${{ hashFiles('ui/**/*.ts', 'ui/**/*.tsx', 'ui/**/*.js', 'ui/**/*.jsx') }} - restore-keys: | - ${{ runner.os }}-pnpm-nextjs-${{ hashFiles('ui/pnpm-lock.yaml') }}- - ${{ runner.os }}-pnpm-nextjs- - - name: Install UI dependencies - working-directory: ./ui - run: pnpm install --frozen-lockfile --prefer-offline - - name: Build UI application - working-directory: ./ui - run: pnpm run build - - name: Cache Playwright browsers - uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1 - id: playwright-cache - with: - path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('ui/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-playwright- - - name: Install Playwright browsers - working-directory: ./ui - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: pnpm run test:e2e:install - - name: Run E2E tests - working-directory: ./ui - run: pnpm run test:e2e - - name: Upload test reports - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: failure() - with: - name: playwright-report - path: ui/playwright-report/ - retention-days: 30 - - name: Cleanup services - if: always() - run: | - echo "Shutting down services..." - docker compose down -v || true - echo "Cleanup completed" diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md index 3a667853a7..7ac2c70086 100644 --- a/ui/CHANGELOG.md +++ b/ui/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to the **Prowler UI** are documented in this file. +## [1.18.1] (Prowler UNRELEASED) + +### 🐞 Fixed + +- Scans page polling now only refreshes scan table data instead of re-rendering the entire server component tree, eliminating redundant API calls to providers, findings, and compliance endpoints every 5 seconds + +--- + ## [1.18.0] (Prowler v5.18.0) ### 🔄 Changed diff --git a/ui/app/(prowler)/scans/page.tsx b/ui/app/(prowler)/scans/page.tsx index c3f1489714..2d0416f37c 100644 --- a/ui/app/(prowler)/scans/page.tsx +++ b/ui/app/(prowler)/scans/page.tsx @@ -1,21 +1,19 @@ import { Suspense } from "react"; import { getAllProviders } from "@/actions/providers"; -import { getScans, getScansByState } from "@/actions/scans"; +import { getScans } from "@/actions/scans"; import { auth } from "@/auth.config"; import { MutedFindingsConfigButton } from "@/components/providers"; import { - AutoRefresh, NoProvidersAdded, NoProvidersConnected, ScansFilters, } from "@/components/scans"; import { LaunchScanWorkflow } from "@/components/scans/launch-workflow"; import { SkeletonTableScans } from "@/components/scans/table"; -import { ColumnGetScans } from "@/components/scans/table/scans"; +import { ScansTableWithPolling } from "@/components/scans/table/scans"; import { ContentLayout } from "@/components/ui"; import { CustomBanner } from "@/components/ui/custom/custom-banner"; -import { DataTable } from "@/components/ui/table"; import { createProviderDetailsMapping, extractProviderUIDs, @@ -57,15 +55,6 @@ export default async function Scans({ const hasManageScansPermission = session?.user?.permissions?.manage_scans; - // Get scans data to check for executing scans - const scansData = await getScansByState(); - - const hasExecutingScan = scansData?.data?.some( - (scan: ScanProps) => - scan.attributes.state === "executing" || - scan.attributes.state === "available", - ); - // Extract provider UIDs and create provider details mapping for filtering const providerUIDs = providersData ? extractProviderUIDs(providersData) : []; const providerDetails = providersData @@ -82,7 +71,6 @@ export default async function Scans({ return ( - <> <> {!hasManageScansPermission ? ( @@ -177,11 +165,10 @@ const SSRDataTableScans = async ({ }) || []; return ( - ); }; diff --git a/ui/components/scans/launch-workflow/launch-scan-workflow-form.tsx b/ui/components/scans/launch-workflow/launch-scan-workflow-form.tsx index 7fb31d7e2a..b0047445c7 100644 --- a/ui/components/scans/launch-workflow/launch-scan-workflow-form.tsx +++ b/ui/components/scans/launch-workflow/launch-scan-workflow-form.tsx @@ -13,6 +13,7 @@ import { Form } from "@/components/ui/form"; import { toast } from "@/components/ui/toast"; import { onDemandScanFormSchema } from "@/types"; +import { SCAN_LAUNCHED_EVENT } from "../table/scans/scans-table-with-polling"; import { SelectScanProvider } from "./select-scan-provider"; type ProviderInfo = { @@ -85,6 +86,8 @@ export const LaunchScanWorkflow = ({ }); // Reset form after successful submission form.reset(); + // Notify the scans table to refresh and pick up the new scan + window.dispatchEvent(new Event(SCAN_LAUNCHED_EVENT)); } }; diff --git a/ui/components/scans/table/scans/index.ts b/ui/components/scans/table/scans/index.ts index 2567c0098c..18c3035f7d 100644 --- a/ui/components/scans/table/scans/index.ts +++ b/ui/components/scans/table/scans/index.ts @@ -1,3 +1,4 @@ export * from "./column-get-scans"; export * from "./data-table-row-actions"; export * from "./data-table-row-details"; +export * from "./scans-table-with-polling"; diff --git a/ui/components/scans/table/scans/scans-table-with-polling.tsx b/ui/components/scans/table/scans/scans-table-with-polling.tsx new file mode 100644 index 0000000000..5c59a5f267 --- /dev/null +++ b/ui/components/scans/table/scans/scans-table-with-polling.tsx @@ -0,0 +1,126 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; + +import { getScans } from "@/actions/scans"; +import { AutoRefresh } from "@/components/scans"; +import { DataTable } from "@/components/ui/table"; +import { MetaDataProps, ScanProps, SearchParamsProps } from "@/types"; + +import { ColumnGetScans } from "./column-get-scans"; + +export const SCAN_LAUNCHED_EVENT = "scan-launched"; + +interface ScansTableWithPollingProps { + initialData: ScanProps[]; + initialMeta?: MetaDataProps; + searchParams: SearchParamsProps; +} + +const EXECUTING_STATES = ["executing", "available"] as const; + +function expandScansWithProviderInfo( + scans: ScanProps[], + included?: Array<{ type: string; id: string; attributes: any }>, +) { + return ( + scans?.map((scan) => { + const providerId = scan.relationships?.provider?.data?.id; + + if (!providerId) { + return { ...scan, providerInfo: undefined }; + } + + const providerData = included?.find( + (item) => item.type === "providers" && item.id === providerId, + ); + + if (!providerData) { + return { ...scan, providerInfo: undefined }; + } + + return { + ...scan, + providerInfo: { + provider: providerData.attributes.provider, + uid: providerData.attributes.uid, + alias: providerData.attributes.alias, + }, + }; + }) || [] + ); +} + +export function ScansTableWithPolling({ + initialData, + initialMeta, + searchParams, +}: ScansTableWithPollingProps) { + const [scansData, setScansData] = useState(initialData); + const [meta, setMeta] = useState(initialMeta); + + const hasExecutingScan = scansData.some((scan) => + EXECUTING_STATES.includes( + scan.attributes.state as (typeof EXECUTING_STATES)[number], + ), + ); + + const handleRefresh = useCallback(async () => { + const page = parseInt(searchParams.page?.toString() || "1", 10); + const pageSize = parseInt(searchParams.pageSize?.toString() || "10", 10); + const sort = searchParams.sort?.toString(); + + const filters = Object.fromEntries( + Object.entries(searchParams).filter( + ([key]) => key.startsWith("filter[") && key !== "scanId", + ), + ); + + const query = (filters["filter[search]"] as string) || ""; + + const result = await getScans({ + query, + page, + sort, + filters, + pageSize, + include: "provider", + }); + + if (result?.data) { + const expanded = expandScansWithProviderInfo( + result.data, + result.included, + ); + setScansData(expanded); + + if (result && "meta" in result) { + setMeta(result.meta as MetaDataProps); + } + } + }, [searchParams]); + + // Listen for scan launch events to trigger an immediate refresh + useEffect(() => { + const handler = () => { + handleRefresh(); + }; + window.addEventListener(SCAN_LAUNCHED_EVENT, handler); + return () => window.removeEventListener(SCAN_LAUNCHED_EVENT, handler); + }, [handleRefresh]); + + return ( + <> + + + + ); +} diff --git a/ui/components/ui/custom/custom-table-link.tsx b/ui/components/ui/custom/custom-table-link.tsx index 80b6b000bf..9fba1e2c55 100644 --- a/ui/components/ui/custom/custom-table-link.tsx +++ b/ui/components/ui/custom/custom-table-link.tsx @@ -21,7 +21,9 @@ export const TableLink = ({ href, label, isDisabled }: TableLinkProps) => { return ( ); }; diff --git a/ui/tests/scans/scans-page.ts b/ui/tests/scans/scans-page.ts index e3e19804fc..6f0052873d 100644 --- a/ui/tests/scans/scans-page.ts +++ b/ui/tests/scans/scans-page.ts @@ -105,6 +105,7 @@ export class ScansPage extends BasePage { await expect(this.scanTable).toBeVisible(); // Find a row that contains the account ID (provider UID in Cloud Provider column) + // Note: Use a more specific locator strategy if possible in the future const rowWithAccountId = this.scanTable .locator("tbody tr") .filter({ hasText: accountId })