mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-10-14 15:34:19 +00:00
* chore(Celery): add basic Celery worker with broker only * chore(Celery): saving progress. Not able to schedule tasks * fix(Celery): add celery app for use by django * fix(Celery): register tasks * fix(Docker): add celery workers to docker-compose * chore(Celery): add django-celery-results backend to store results using Django ORM * fix(Celery): get app config the correct way * fix(Docker): start connecting docker Celery workers to Valkey not yet operational * fix(Celery): get celery & django to work in docker-compose * docs(Celery): document how to run Celery in development environment includes changes to support the configuration and deployment of Celery worker and its dependencies, Valkey and Postgres. * fix(GHA): add valkey to CI services * fix(GHA): add valkey to CI services * fix(GHA): add valkey-cli ping to CI services * fix(GHA): use right port for valkey * fix(Views): remove debug task code * test(Celery): start adding celery task tests not yet working! * fix(pyproject): rollback django upgrade * fix(docker): updated docker runtime and env vars based on feedback from #20 * fix(Dockerfile): include dependencies for psutil psutil was introduced by pytest-celery * fix(Backend): PRWLR-4013 fix celery settings structure * fix(Celery): update celery app to work with new settings structure * fix(Views): remove debug task code * fix(Config): remove debug code * fix(Celery): update celery app name when running worker --------- Co-authored-by: Víctor Fernández Poyatos <victor@prowler.com>
140 lines
4.6 KiB
YAML
140 lines
4.6 KiB
YAML
name: pr-lint-test
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- "main"
|
|
pull_request:
|
|
branches:
|
|
- "main"
|
|
|
|
|
|
env:
|
|
POSTGRES_HOST: localhost
|
|
POSTGRES_PORT: 5432
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: prowler_db_test
|
|
VALKEY_HOST: localhost
|
|
VALKEY_PORT: 6379
|
|
VALKEY_DB: 0
|
|
|
|
|
|
jobs:
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
python-version: ["3.12"]
|
|
|
|
# Service containers to run with `test`
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
image: postgres
|
|
env:
|
|
POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
|
|
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
|
|
POSTGRES_USER: ${{ env.POSTGRES_USER }}
|
|
POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
|
|
POSTGRES_DB: ${{ env.POSTGRES_DB }}
|
|
# Set health checks to wait until postgres has started
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
valkey:
|
|
image: valkey/valkey:7-alpine3.19
|
|
env:
|
|
VALKEY_HOST: ${{ env.VALKEY_HOST }}
|
|
VALKEY_PORT: ${{ env.VALKEY_PORT }}
|
|
VALKEY_DB: ${{ env.VALKEY_DB }}
|
|
# Set health checks to wait until postgres has started
|
|
ports:
|
|
- 6379:6379
|
|
options: >-
|
|
--health-cmd "valkey-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Test if changes are in not ignored paths
|
|
id: are-non-ignored-files-changed
|
|
uses: tj-actions/changed-files@v42
|
|
with:
|
|
files: ./**
|
|
files_ignore: |
|
|
.github/**
|
|
README.md
|
|
docs/**
|
|
permissions/**
|
|
mkdocs.yml
|
|
- name: Install poetry
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pipx install poetry
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
cache: "poetry"
|
|
- name: Install dependencies
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry install
|
|
poetry run pip list
|
|
VERSION=$(curl --silent "https://api.github.com/repos/hadolint/hadolint/releases/latest" | \
|
|
grep '"tag_name":' | \
|
|
sed -E 's/.*"v([^"]+)".*/\1/' \
|
|
) && curl -L -o /tmp/hadolint "https://github.com/hadolint/hadolint/releases/download/v${VERSION}/hadolint-Linux-x86_64" \
|
|
&& chmod +x /tmp/hadolint
|
|
|
|
- name: Poetry check
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry lock --check
|
|
- name: Lint with ruff
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run ruff check . --exclude contrib
|
|
- name: Check Format with ruff
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run ruff format --check . --exclude contrib
|
|
- name: Lint with pylint
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run pylint --disable=W,C,R,E -j 0 -rn -sn src/
|
|
- name: Bandit
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run bandit -q -lll -x '*_test.py,./contrib/' -r .
|
|
- name: Safety
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run safety check --ignore 70612
|
|
- name: Vulture
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run vulture --exclude "contrib" --min-confidence 100 .
|
|
- name: Hadolint
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
/tmp/hadolint Dockerfile --ignore=DL3013
|
|
- name: Test with pytest
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
run: |
|
|
poetry run pytest -n auto --cov=./src/backend --cov-report=xml src/backend
|
|
- name: Upload coverage reports to Codecov
|
|
if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true'
|
|
uses: codecov/codecov-action@v4
|
|
env:
|
|
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|