diff --git a/.github/actions/setup-python-poetry/action.yml b/.github/actions/setup-python-poetry/action.yml index 3b774262f9..790f3ef0e6 100644 --- a/.github/actions/setup-python-poetry/action.yml +++ b/.github/actions/setup-python-poetry/action.yml @@ -22,8 +22,8 @@ inputs: runs: using: 'composite' steps: - - name: Replace @master with current branch in pyproject.toml - if: github.event_name == 'pull_request' && github.base_ref == 'master' + - name: Replace @master with current branch in pyproject.toml (prowler repo only) + if: github.event_name == 'pull_request' && github.base_ref == 'master' && github.repository == 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} run: | @@ -37,8 +37,8 @@ runs: python -m pip install --upgrade pip pipx install poetry==${{ inputs.poetry-version }} - - name: Update SDK resolved_reference to latest commit - if: github.event_name == 'push' && github.ref == 'refs/heads/master' + - name: Update poetry.lock with latest Prowler commit + if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} run: | @@ -50,7 +50,21 @@ runs: echo "Updated resolved_reference:" grep -A2 -B2 "resolved_reference" poetry.lock - - name: Update poetry.lock + - name: Update SDK resolved_reference to latest commit (prowler repo on push) + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + LATEST_COMMIT=$(curl -s "https://api.github.com/repos/prowler-cloud/prowler/commits/master" | jq -r '.sha') + echo "Latest commit hash: $LATEST_COMMIT" + sed -i '/url = "https:\/\/github\.com\/prowler-cloud\/prowler\.git"/,/resolved_reference = / { + s/resolved_reference = "[a-f0-9]\{40\}"/resolved_reference = "'"$LATEST_COMMIT"'"/ + }' poetry.lock + echo "Updated resolved_reference:" + grep -A2 -B2 "resolved_reference" poetry.lock + + - name: Update poetry.lock (prowler repo only) + if: github.repository == 'prowler-cloud/prowler' shell: bash working-directory: ${{ inputs.working-directory }} run: poetry lock @@ -69,3 +83,11 @@ runs: run: | poetry install --no-root poetry run pip list + + - name: Update Prowler Cloud API Client + if: github.repository_owner == 'prowler-cloud' && github.repository != 'prowler-cloud/prowler' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry remove prowler-cloud-api-client + poetry add ./prowler-cloud-api-client diff --git a/.github/workflows/api-code-quality.yml b/.github/workflows/api-code-quality.yml new file mode 100644 index 0000000000..2c52e532ba --- /dev/null +++ b/.github/workflows/api-code-quality.yml @@ -0,0 +1,74 @@ +name: 'API: Code Quality' + +on: + push: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-code-quality.yml' + pull_request: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-code-quality.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + +jobs: + api-code-quality: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Poetry check + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry check --lock + + - name: Ruff lint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run ruff check . --exclude contrib + + - name: Ruff format + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run ruff format --check . --exclude contrib + + - name: Pylint + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/ diff --git a/.github/workflows/api-container-checks.yml b/.github/workflows/api-container-checks.yml new file mode 100644 index 0000000000..0b9f5a28bc --- /dev/null +++ b/.github/workflows/api-container-checks.yml @@ -0,0 +1,94 @@ +name: 'API: Container Checks' + +on: + push: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-container-checks.yml' + pull_request: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-container-checks.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + IMAGE_NAME: prowler-api + +jobs: + api-dockerfile-lint: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check if Dockerfile changed + id: dockerfile-changed + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files: api/Dockerfile + + - name: Lint Dockerfile with Hadolint + if: steps.dockerfile-changed.outputs.any_changed == 'true' + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: api/Dockerfile + ignore: DL3013 + + api-container-build-and-scan: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Set up Docker Buildx + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Build container + if: steps.check-changes.outputs.any_changed == 'true' + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ env.API_WORKING_DIR }} + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan container with Trivy + if: github.repository == 'prowler-cloud/prowler' && steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/trivy-scan + with: + image-name: ${{ env.IMAGE_NAME }} + image-tag: ${{ github.sha }} + fail-on-critical: 'false' + severity: 'CRITICAL' diff --git a/.github/workflows/api-pull-request.yml b/.github/workflows/api-pull-request.yml deleted file mode 100644 index d838fd489a..0000000000 --- a/.github/workflows/api-pull-request.yml +++ /dev/null @@ -1,255 +0,0 @@ -name: 'API: Pull Request' - -on: - push: - branches: - - 'master' - - 'v5.*' - paths: - - 'api/**' - - '.github/workflows/api-pull-request.yml' - pull_request: - branches: - - 'master' - - 'v5.*' - paths: - - 'api/**' - - '.github/workflows/api-pull-request.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 - POSTGRES_ADMIN_USER: prowler - POSTGRES_ADMIN_PASSWORD: S3cret - POSTGRES_USER: prowler_user - POSTGRES_PASSWORD: prowler - POSTGRES_DB: postgres-db - VALKEY_HOST: localhost - VALKEY_PORT: 6379 - VALKEY_DB: 0 - API_WORKING_DIR: ./api - IMAGE_NAME: prowler-api - -jobs: - check-changes: - if: github.repository == 'prowler-cloud/prowler' - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - api-changed: ${{ steps.filter.outputs.api }} - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Check for API changes - id: filter - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files_ignore: | - api/docs/** - api/README.md - api/CHANGELOG.md - - - name: Set output - id: set-output - run: | - if [[ "${{ steps.filter.outputs.any_changed }}" == "true" ]]; then - echo "api=true" >> $GITHUB_OUTPUT - else - echo "api=false" >> $GITHUB_OUTPUT - fi - - code-quality: - needs: check-changes - if: github.repository == 'prowler-cloud/prowler' && needs.check-changes.outputs.api-changed == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - strategy: - matrix: - python-version: - - '3.12' - defaults: - run: - working-directory: ./api - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python with Poetry - uses: ./.github/actions/setup-python-poetry - with: - python-version: ${{ matrix.python-version }} - working-directory: ./api - - - name: Poetry check - run: poetry check --lock - - - name: Ruff lint - run: poetry run ruff check . --exclude contrib - - - name: Ruff format - run: poetry run ruff format --check . --exclude contrib - - - name: Pylint - run: poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/ - - security-scans: - needs: check-changes - if: github.repository == 'prowler-cloud/prowler' && needs.check-changes.outputs.api-changed == 'true' - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - strategy: - matrix: - python-version: - - '3.12' - defaults: - run: - working-directory: ./api - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python with Poetry - uses: ./.github/actions/setup-python-poetry - with: - python-version: ${{ matrix.python-version }} - working-directory: ./api - - - name: Bandit - run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r . - - - name: Safety - # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API - # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X - run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 - - - name: Vulture - run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 . - - tests: - needs: check-changes - if: github.repository == 'prowler-cloud/prowler' && needs.check-changes.outputs.api-changed == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - strategy: - matrix: - python-version: - - '3.12' - defaults: - run: - working-directory: ./api - - services: - postgres: - image: postgres - env: - POSTGRES_HOST: ${{ env.POSTGRES_HOST }} - POSTGRES_PORT: ${{ env.POSTGRES_PORT }} - POSTGRES_USER: ${{ env.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} - POSTGRES_DB: ${{ env.POSTGRES_DB }} - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - valkey: - image: valkey/valkey:7-alpine3.19 - env: - VALKEY_HOST: ${{ env.VALKEY_HOST }} - VALKEY_PORT: ${{ env.VALKEY_PORT }} - VALKEY_DB: ${{ env.VALKEY_DB }} - ports: - - 6379:6379 - options: >- - --health-cmd "valkey-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Setup Python with Poetry - uses: ./.github/actions/setup-python-poetry - with: - python-version: ${{ matrix.python-version }} - working-directory: ./api - - - name: Run tests with pytest - run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - flags: api - - dockerfile-lint: - needs: check-changes - if: github.repository == 'prowler-cloud/prowler' && needs.check-changes.outputs.api-changed == 'true' - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Lint Dockerfile with Hadolint - uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 - with: - dockerfile: api/Dockerfile - ignore: DL3013 - - container-build-and-scan: - needs: check-changes - if: github.repository == 'prowler-cloud/prowler' && needs.check-changes.outputs.api-changed == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - security-events: write - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Build container - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: ${{ env.API_WORKING_DIR }} - push: false - load: true - tags: ${{ env.IMAGE_NAME }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Scan container with Trivy - uses: ./.github/actions/trivy-scan - with: - image-name: ${{ env.IMAGE_NAME }} - image-tag: ${{ github.sha }} - fail-on-critical: 'false' - severity: 'CRITICAL' diff --git a/.github/workflows/api-security.yml b/.github/workflows/api-security.yml new file mode 100644 index 0000000000..126b957ed0 --- /dev/null +++ b/.github/workflows/api-security.yml @@ -0,0 +1,72 @@ +name: 'API: Security' + +on: + push: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-security.yml' + pull_request: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-security.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + API_WORKING_DIR: ./api + +jobs: + api-security-scans: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Bandit + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run bandit -q -lll -x '*_test.py,./contrib/' -r . + + - name: Safety + if: steps.check-changes.outputs.any_changed == 'true' + # 76352, 76353, 77323 come from SDK, but they cannot upgrade it yet. It does not affect API + # TODO: Botocore needs urllib3 1.X so we need to ignore these vulnerabilities 77744,77745. Remove this once we upgrade to urllib3 2.X + run: poetry run safety check --ignore 70612,66963,74429,76352,76353,77323,77744,77745 + + - name: Vulture + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run vulture --exclude "contrib,tests,conftest.py" --min-confidence 100 . diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml new file mode 100644 index 0000000000..3f827d8283 --- /dev/null +++ b/.github/workflows/api-tests.yml @@ -0,0 +1,110 @@ +name: 'API: Tests' + +on: + push: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-tests.yml' + pull_request: + branches: + - 'master' + - 'v5.*' + paths: + - 'api/**' + - '.github/workflows/api-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + POSTGRES_ADMIN_USER: prowler + POSTGRES_ADMIN_PASSWORD: S3cret + POSTGRES_USER: prowler_user + POSTGRES_PASSWORD: prowler + POSTGRES_DB: postgres-db + VALKEY_HOST: localhost + VALKEY_PORT: 6379 + VALKEY_DB: 0 + API_WORKING_DIR: ./api + +jobs: + api-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + strategy: + matrix: + python-version: + - '3.12' + defaults: + run: + working-directory: ./api + + services: + postgres: + image: postgres + env: + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + POSTGRES_PORT: ${{ env.POSTGRES_PORT }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ env.POSTGRES_DB }} + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + valkey: + image: valkey/valkey:7-alpine3.19 + env: + VALKEY_HOST: ${{ env.VALKEY_HOST }} + VALKEY_PORT: ${{ env.VALKEY_PORT }} + VALKEY_DB: ${{ env.VALKEY_DB }} + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Check for API changes + id: check-changes + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files_ignore: | + api/docs/** + api/README.md + api/CHANGELOG.md + + - name: Setup Python with Poetry + if: steps.check-changes.outputs.any_changed == 'true' + uses: ./.github/actions/setup-python-poetry + with: + python-version: ${{ matrix.python-version }} + working-directory: ./api + + - name: Run tests with pytest + if: steps.check-changes.outputs.any_changed == 'true' + run: poetry run pytest --cov=./src/backend --cov-report=xml src/backend + + - name: Upload coverage reports to Codecov + if: steps.check-changes.outputs.any_changed == 'true' + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: api