diff --git a/Dockerfile b/Dockerfile index 9bda1950ec..5c34c3d794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,46 +1,39 @@ -FROM python:3.12-alpine AS build +# Base image for building the application +FROM python:3.12-alpine AS base LABEL maintainer="https://github.com/prowler-cloud/api" +# Install necessary dependencies # hadolint ignore=DL3018 RUN apk --no-cache add gcc python3-dev musl-dev linux-headers curl-dev RUN apk --no-cache upgrade && \ addgroup -g 1000 prowler && \ adduser -D -u 1000 -G prowler prowler -USER prowler WORKDIR /home/prowler +USER prowler +# Install Poetry and project dependencies COPY pyproject.toml ./ - -RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir poetry - -COPY src/backend/ ./backend/ - +RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir poetry ENV PATH="/home/prowler/.local/bin:$PATH" +RUN poetry install && rm -rf ~/.cache/pip ~/.cache/pypoetry -RUN poetry install && \ - rm -rf ~/.cache/pip - -COPY docker-entrypoint.sh ./docker-entrypoint.sh +# Copy backend source code and entrypoint script +COPY src/backend/ ./backend/ +COPY docker-entrypoint.sh /home/prowler/docker-entrypoint.sh WORKDIR /home/prowler/backend -# Development image -# hadolint ignore=DL3006 -FROM build AS dev - +# Development stage +FROM base AS dev USER 0 # hadolint ignore=DL3018 RUN apk --no-cache add curl vim - USER prowler +ENTRYPOINT ["/home/prowler/docker-entrypoint.sh", "dev"] -ENTRYPOINT ["../docker-entrypoint.sh", "dev"] - -# Production image -FROM build - -ENTRYPOINT ["../docker-entrypoint.sh", "prod"] +# Production stage +FROM base AS prod +ENTRYPOINT ["/home/prowler/docker-entrypoint.sh", "prod"] diff --git a/README.md b/README.md index 890ed943f9..17f5604125 100644 --- a/README.md +++ b/README.md @@ -21,53 +21,13 @@ This repository contains the JSON API and Task Runner components for Prowler, which facilitate a complete backend that interacts with the Prowler SDK and is used by the Prowler UI. -# Production deployment - -## Install all dependencies with Poetry - -```console -poetry install -poetry shell -``` - -## Modify environment variables - -Under the root path of the project, you can find a file called `.env.example`. This file shows all the environment variables that the project uses. You can *must* create a new file called `.env` and set the values for the variables. - -Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the Poetry interpreter, not before. Otherwise, variables will not be loaded properly. - -## Run migrations - -For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run: - -```console -python manage.py migrate --database admin -``` - -## Run the Celery worker - -```console -cd src/backend -python -m celery -A config.celery worker -l info -E -``` - -## Run the Django server with Gunicorn - -```console -cd src/backend -gunicorn -c backend/guniconf.py backend.wsgi:application -``` - -> By default, the Gunicorn server will try to use as many workers as your machine can handle. You can manually change that in the `src/backend/backend/guniconf.py` file. - -# ๐Ÿ’ป Development guide - +# Components The Prowler API is composed of the following components: -- The JSON API, which is the main component of the API. +- The JSON API, which is an API built with Django Rest Framework. - The Celery worker, which is responsible for executing the background tasks that are defined in the JSON API. - The PostgreSQL database, which is used to store the data. -- The Valkey database, which is used to manage the background tasks. +- The Valkey database, which is an in-memory database which is used as a message broker for the Celery workers. ## Note about Valkey @@ -75,9 +35,25 @@ The Prowler API is composed of the following components: Valkey exposes a Redis 7.2 compliant API. Any service that exposes the Redis API can be used with Prowler API. -## Local deployment +# Modify environment variables -This method requires installing a Python virtual environment and keep dependencies updated. +Under the root path of the project, you can find a file called `.env.example`. This file shows all the environment variables that the project uses. You *must* create a new file called `.env` and set the values for the variables. + +## Local deployment +Keep in mind if you export the `.env` file to use it with local deployment that you will have to do it within the context of the Poetry interpreter, not before. Otherwise, variables will not be loaded properly. + +To do this, you can run: + +```console +poetry shell +set -a +source .env +``` + +# ๐Ÿš€ Production deployment +## Docker deployment + +This method requires `docker` and `docker compose`. ### Clone the repository @@ -90,9 +66,113 @@ git clone git@github.com:prowler-cloud/api.git ``` -### Start the PostgreSQL database and Valkey +### Build the base image + +```console +docker compose --profile prod build +``` + +### Run the production service + +This command will start the Django production server and the Celery worker and also the Valkey and PostgreSQL databases. + +```console +docker compose --profile prod up -d +``` + +You can access the server in `http://localhost:8080`. + +> **NOTE:** notice how the port is different. When developing using docker, the port will be `8080` to prevent conflicts. + +### View the Production Server Logs + +To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern: + +```console +docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-') + +## Local deployment + +To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `poetry` and `docker compose` are installed. + +### Clone the repository + +```console +# HTTPS +git clone https://github.com/prowler-cloud/api.git + +# SSH +git clone git@github.com:prowler-cloud/api.git + +``` +### Install all dependencies with Poetry + +```console +poetry install +poetry shell +``` + +## Start the PostgreSQL Database and Valkey + +The PostgreSQL database (version 16.3) and Valkey (version 7) are required for the development environment. To make development easier, we have provided a `docker-compose` file that will start these components for you. + +**Note:** Make sure to use the specified versions, as there are features in our setup that may not be compatible with older versions of PostgreSQL and Valkey. + + +```console +docker compose up postgres valkey -d +``` + +## Deploy Django and the Celery worker + +### Run migrations + +For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run: + +```console +cd src/backend +python manage.py migrate --database admin +``` + +### Run the Celery worker + +```console +cd src/backend +python -m celery -A config.celery worker -l info -E +``` + +### Run the Django server with Gunicorn + +```console +cd src/backend +gunicorn -c config/guniconf.py config.wsgi:application +``` + +> By default, the Gunicorn server will try to use as many workers as your machine can handle. You can manually change that in the `src/backend/config/guniconf.py` file. + +# ๐Ÿงช Development guide + +## Local deployment + +To use this method, you'll need to set up a Python virtual environment (version ">=3.11,<3.13") and keep dependencies updated. Additionally, ensure that `poetry` and `docker compose` are installed. + +### Clone the repository + +```console +# HTTPS +git clone https://github.com/prowler-cloud/api.git + +# SSH +git clone git@github.com:prowler-cloud/api.git + +``` + +### Start the PostgreSQL Database and Valkey + +The PostgreSQL database (version 16.3) and Valkey (version 7) are required for the development environment. To make development easier, we have provided a `docker-compose` file that will start these components for you. + +**Note:** Make sure to use the specified versions, as there are features in our setup that may not be compatible with older versions of PostgreSQL and Valkey. -The PostgreSQL database and Valkey are required for the development environment. To make development easier, we have provided a docker-compose file that will start them for you. ```console docker compose up postgres valkey -d @@ -112,14 +192,14 @@ poetry shell For migrations, you need to force the `admin` database router. Assuming you have the correct environment variables and Python virtual environment, run: ```console +cd src/backend python manage.py migrate --database admin ``` ### Run the Django development server ```console -cd backend -python manage.py migrate --database admin +cd src/backend python manage.py runserver ``` @@ -170,17 +250,10 @@ All changes in the code will be automatically reloaded in the server. ### View the development server logs -For Django +To view the logs for any component (e.g., Django, Celery worker), you can use the following command with a wildcard. This command will follow logs for any container that matches the specified pattern: ```console -docker logs -f api-api-dev-1 -``` - -or for the Celery worker: - -```console -docker logs -f api-worker-dev-1 -``` +docker logs -f $(docker ps --format "{{.Names}}" | grep 'api-') ## Applying migrations @@ -188,6 +261,7 @@ For migrations, you need to force the `admin` database router. Assuming you have ```console poetry shell +cd src/backend python manage.py migrate --database admin ``` @@ -197,7 +271,7 @@ Fixtures are used to populate the database with initial development data. ```console poetry shell -# For dev users +cd src/backend python manage.py loaddata api/fixtures/0_dev_users.json --database admin ```