mirror of
https://github.com/prowler-cloud/prowler.git
synced 2026-03-24 04:28:02 +00:00
Compare commits
51 Commits
5.20.0
...
PRWLR-7835
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0c669a069b | ||
|
|
4cb2778bfa | ||
|
|
dfe2b43ea5 | ||
|
|
c42351335d | ||
|
|
f5d73c2224 | ||
|
|
e981e90f89 | ||
|
|
59255c0b76 | ||
|
|
c64c86d8a3 | ||
|
|
28ce5c9b39 | ||
|
|
9fe53cf13d | ||
|
|
77fa5d1644 | ||
|
|
a77297ff4c | ||
|
|
8e5bd7db18 | ||
|
|
f1f6120295 | ||
|
|
201c5211fe | ||
|
|
6ebc67b776 | ||
|
|
1a0fac0db8 | ||
|
|
8ac481f610 | ||
|
|
cfa4260c4b | ||
|
|
bc60a35318 | ||
|
|
4c078e8d66 | ||
|
|
241bd3e437 | ||
|
|
7669f00095 | ||
|
|
218f8511f5 | ||
|
|
d70e8f0633 | ||
|
|
e30101b789 | ||
|
|
dd7fed06ac | ||
|
|
29ab107edf | ||
|
|
a56686bf1a | ||
|
|
fb870711d9 | ||
|
|
774211df2b | ||
|
|
0f06ec2593 | ||
|
|
3e92af063d | ||
|
|
cc4d6726b4 | ||
|
|
b5ac1eca73 | ||
|
|
6bec69fa81 | ||
|
|
2c71a15a12 | ||
|
|
ed12d2f418 | ||
|
|
5602a4182d | ||
|
|
f8fde7018c | ||
|
|
41b95da6e6 | ||
|
|
68e2343ca2 | ||
|
|
f86b73190b | ||
|
|
710df66fc4 | ||
|
|
a7c06a88e8 | ||
|
|
8252fddd7a | ||
|
|
2d25dbf2b5 | ||
|
|
042d8aefea | ||
|
|
5c691028b9 | ||
|
|
cacef6f0cd | ||
|
|
a02cc6364b |
@@ -14,6 +14,7 @@ All notable changes to the **Prowler API** are documented in this file.
|
||||
- Support for `passed_findings` and `total_findings` fields in compliance requirement overview for accurate Prowler ThreatScore calculation [(#8582)](https://github.com/prowler-cloud/prowler/pull/8582)
|
||||
- PDF reporting for Prowler ThreatScore [(#8867)](https://github.com/prowler-cloud/prowler/pull/8867)
|
||||
- Database read replica support [(#8869)](https://github.com/prowler-cloud/prowler/pull/8869)
|
||||
- Support for configuring multiple LLM providers [(#8772)](https://github.com/prowler-cloud/prowler/pull/8772)
|
||||
- Support Common Cloud Controls for AWS, Azure and GCP [(#8000)](https://github.com/prowler-cloud/prowler/pull/8000)
|
||||
- Add `provider_id__in` filter support to findings and findings severity overview endpoints [(#8951)](https://github.com/prowler-cloud/prowler/pull/8951)
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ from api.models import (
|
||||
Finding,
|
||||
Integration,
|
||||
Invitation,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
Membership,
|
||||
OverviewStatusChoices,
|
||||
PermissionChoices,
|
||||
@@ -928,3 +930,45 @@ class TenantApiKeyFilter(FilterSet):
|
||||
"revoked": ["exact"],
|
||||
"name": ["exact", "icontains"],
|
||||
}
|
||||
|
||||
|
||||
class LighthouseProviderConfigFilter(FilterSet):
|
||||
provider_type = ChoiceFilter(
|
||||
choices=LighthouseProviderConfiguration.LLMProviderChoices.choices
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
choices=LighthouseProviderConfiguration.LLMProviderChoices.choices,
|
||||
field_name="provider_type",
|
||||
lookup_expr="in",
|
||||
)
|
||||
is_active = BooleanFilter()
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderConfiguration
|
||||
fields = {
|
||||
"provider_type": ["exact", "in"],
|
||||
"is_active": ["exact"],
|
||||
}
|
||||
|
||||
|
||||
class LighthouseProviderModelsFilter(FilterSet):
|
||||
provider_type = ChoiceFilter(
|
||||
choices=LighthouseProviderConfiguration.LLMProviderChoices.choices,
|
||||
field_name="provider_configuration__provider_type",
|
||||
)
|
||||
provider_type__in = ChoiceInFilter(
|
||||
choices=LighthouseProviderConfiguration.LLMProviderChoices.choices,
|
||||
field_name="provider_configuration__provider_type",
|
||||
lookup_expr="in",
|
||||
)
|
||||
|
||||
# Allow filtering by model id
|
||||
model_id = CharFilter(field_name="model_id", lookup_expr="exact")
|
||||
model_id__icontains = CharFilter(field_name="model_id", lookup_expr="icontains")
|
||||
model_id__in = CharInFilter(field_name="model_id", lookup_expr="in")
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderModels
|
||||
fields = {
|
||||
"model_id": ["exact", "icontains", "in"],
|
||||
}
|
||||
|
||||
266
api/src/backend/api/migrations/0050_lighthouse_multi_llm.py
Normal file
266
api/src/backend/api/migrations/0050_lighthouse_multi_llm.py
Normal file
@@ -0,0 +1,266 @@
|
||||
# Generated by Django 5.1.12 on 2025-10-09 07:50
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from config.custom_logging import BackendLogger
|
||||
from cryptography.fernet import Fernet
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
import api.rls
|
||||
from api.db_router import MainRouter
|
||||
|
||||
logger = logging.getLogger(BackendLogger.API)
|
||||
|
||||
|
||||
def migrate_lighthouse_configs_forward(apps, schema_editor):
|
||||
"""
|
||||
Migrate data from old LighthouseConfiguration to new multi-provider models.
|
||||
Old system: one LighthouseConfiguration per tenant (always OpenAI).
|
||||
"""
|
||||
LighthouseConfiguration = apps.get_model("api", "LighthouseConfiguration")
|
||||
LighthouseProviderConfiguration = apps.get_model(
|
||||
"api", "LighthouseProviderConfiguration"
|
||||
)
|
||||
LighthouseTenantConfiguration = apps.get_model(
|
||||
"api", "LighthouseTenantConfiguration"
|
||||
)
|
||||
LighthouseProviderModels = apps.get_model("api", "LighthouseProviderModels")
|
||||
|
||||
fernet = Fernet(settings.SECRETS_ENCRYPTION_KEY.encode())
|
||||
|
||||
# Migrate only tenants that actually have a LighthouseConfiguration
|
||||
for old_config in (
|
||||
LighthouseConfiguration.objects.using(MainRouter.admin_db)
|
||||
.select_related("tenant")
|
||||
.all()
|
||||
):
|
||||
tenant = old_config.tenant
|
||||
tenant_id = str(tenant.id)
|
||||
|
||||
try:
|
||||
# Create OpenAI provider configuration for this tenant
|
||||
api_key_decrypted = fernet.decrypt(bytes(old_config.api_key)).decode()
|
||||
credentials_encrypted = fernet.encrypt(
|
||||
json.dumps({"api_key": api_key_decrypted}).encode()
|
||||
)
|
||||
provider_config = LighthouseProviderConfiguration.objects.using(
|
||||
MainRouter.admin_db
|
||||
).create(
|
||||
tenant=tenant,
|
||||
provider_type="openai",
|
||||
credentials=credentials_encrypted,
|
||||
is_active=old_config.is_active,
|
||||
)
|
||||
|
||||
# Create tenant configuration from old values
|
||||
LighthouseTenantConfiguration.objects.using(MainRouter.admin_db).create(
|
||||
tenant=tenant,
|
||||
business_context=old_config.business_context or "",
|
||||
default_provider="openai",
|
||||
default_models={"openai": old_config.model},
|
||||
)
|
||||
|
||||
# Create initial provider model record
|
||||
LighthouseProviderModels.objects.using(MainRouter.admin_db).create(
|
||||
tenant=tenant,
|
||||
provider_configuration=provider_config,
|
||||
model_id=old_config.model,
|
||||
model_name=old_config.model,
|
||||
default_parameters={},
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to migrate lighthouse config for tenant %s", tenant_id
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0049_compliancerequirementoverview_passed_failed_findings"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="LighthouseProviderConfiguration",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"provider_type",
|
||||
models.CharField(
|
||||
choices=[("openai", "OpenAI")],
|
||||
help_text="LLM provider name",
|
||||
max_length=50,
|
||||
),
|
||||
),
|
||||
("base_url", models.URLField(blank=True, null=True)),
|
||||
(
|
||||
"credentials",
|
||||
models.BinaryField(
|
||||
help_text="Encrypted JSON credentials for the provider"
|
||||
),
|
||||
),
|
||||
("is_active", models.BooleanField(default=True)),
|
||||
],
|
||||
options={
|
||||
"db_table": "lighthouse_provider_configurations",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="LighthouseProviderModels",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("model_id", models.CharField(max_length=100)),
|
||||
("model_name", models.CharField(max_length=100)),
|
||||
("default_parameters", models.JSONField(blank=True, default=dict)),
|
||||
],
|
||||
options={
|
||||
"db_table": "lighthouse_provider_models",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="LighthouseTenantConfiguration",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.UUIDField(
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
),
|
||||
),
|
||||
("inserted_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("business_context", models.TextField(blank=True, default="")),
|
||||
("default_provider", models.CharField(blank=True, max_length=50)),
|
||||
("default_models", models.JSONField(blank=True, default=dict)),
|
||||
],
|
||||
options={
|
||||
"db_table": "lighthouse_tenant_config",
|
||||
"abstract": False,
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="lighthouseproviderconfiguration",
|
||||
name="tenant",
|
||||
field=models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="lighthouseprovidermodels",
|
||||
name="provider_configuration",
|
||||
field=models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="available_models",
|
||||
to="api.lighthouseproviderconfiguration",
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="lighthouseprovidermodels",
|
||||
name="tenant",
|
||||
field=models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="lighthousetenantconfiguration",
|
||||
name="tenant",
|
||||
field=models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to="api.tenant"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="lighthouseproviderconfiguration",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "provider_type"], name="lh_pc_tenant_type_idx"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="lighthouseproviderconfiguration",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_lighthouseproviderconfiguration",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="lighthouseproviderconfiguration",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "provider_type"),
|
||||
name="unique_provider_config_per_tenant",
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="lighthouseprovidermodels",
|
||||
index=models.Index(
|
||||
fields=["tenant_id", "provider_configuration"],
|
||||
name="lh_prov_models_cfg_idx",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="lighthouseprovidermodels",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_lighthouseprovidermodels",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="lighthouseprovidermodels",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id", "provider_configuration", "model_id"),
|
||||
name="unique_provider_model_per_configuration",
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="lighthousetenantconfiguration",
|
||||
constraint=api.rls.RowLevelSecurityConstraint(
|
||||
"tenant_id",
|
||||
name="rls_on_lighthousetenantconfiguration",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="lighthousetenantconfiguration",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("tenant_id",), name="unique_tenant_lighthouse_config"
|
||||
),
|
||||
),
|
||||
# Migrate data from old LighthouseConfiguration to new tables
|
||||
# This runs after all tables, indexes, and constraints are created
|
||||
# The old Lighthouse configuration table is not removed, so reverse_code is noop
|
||||
# During rollbacks, the old Lighthouse configuration remains intact while the new tables are removed
|
||||
migrations.RunPython(
|
||||
migrate_lighthouse_configs_forward,
|
||||
reverse_code=migrations.RunPython.noop,
|
||||
),
|
||||
]
|
||||
21
api/src/backend/api/migrations/0051_lighthouse_bedrock.py
Normal file
21
api/src/backend/api/migrations/0051_lighthouse_bedrock.py
Normal file
@@ -0,0 +1,21 @@
|
||||
# Generated by Django 5.1.12 on 2025-10-14 11:46
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("api", "0050_lighthouse_multi_llm"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="lighthouseproviderconfiguration",
|
||||
name="provider_type",
|
||||
field=models.CharField(
|
||||
choices=[("openai", "OpenAI"), ("bedrock", "AWS Bedrock")],
|
||||
help_text="LLM provider name",
|
||||
max_length=50,
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -1873,22 +1873,6 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel):
|
||||
def clean(self):
|
||||
super().clean()
|
||||
|
||||
# Validate temperature
|
||||
if not 0 <= self.temperature <= 1:
|
||||
raise ModelValidationError(
|
||||
detail="Temperature must be between 0 and 1",
|
||||
code="invalid_temperature",
|
||||
pointer="/data/attributes/temperature",
|
||||
)
|
||||
|
||||
# Validate max_tokens
|
||||
if not 500 <= self.max_tokens <= 5000:
|
||||
raise ModelValidationError(
|
||||
detail="Max tokens must be between 500 and 5000",
|
||||
code="invalid_max_tokens",
|
||||
pointer="/data/attributes/max_tokens",
|
||||
)
|
||||
|
||||
@property
|
||||
def api_key_decoded(self):
|
||||
"""Return the decrypted API key, or None if unavailable or invalid."""
|
||||
@@ -1913,15 +1897,6 @@ class LighthouseConfiguration(RowLevelSecurityProtectedModel):
|
||||
code="invalid_api_key",
|
||||
pointer="/data/attributes/api_key",
|
||||
)
|
||||
|
||||
# Validate OpenAI API key format
|
||||
openai_key_pattern = r"^sk-[\w-]+T3BlbkFJ[\w-]+$"
|
||||
if not re.match(openai_key_pattern, value):
|
||||
raise ModelValidationError(
|
||||
detail="Invalid OpenAI API key format.",
|
||||
code="invalid_api_key",
|
||||
pointer="/data/attributes/api_key",
|
||||
)
|
||||
self.api_key = fernet.encrypt(value.encode())
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
@@ -1984,3 +1959,185 @@ class Processor(RowLevelSecurityProtectedModel):
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "processors"
|
||||
|
||||
|
||||
class LighthouseProviderConfiguration(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Per-tenant configuration for an LLM provider (credentials, base URL, activation).
|
||||
|
||||
One configuration per provider type per tenant.
|
||||
"""
|
||||
|
||||
class LLMProviderChoices(models.TextChoices):
|
||||
OPENAI = "openai", _("OpenAI")
|
||||
BEDROCK = "bedrock", _("AWS Bedrock")
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
|
||||
provider_type = models.CharField(
|
||||
max_length=50,
|
||||
choices=LLMProviderChoices.choices,
|
||||
help_text="LLM provider name",
|
||||
)
|
||||
|
||||
# For OpenAI-compatible providers
|
||||
base_url = models.URLField(blank=True, null=True)
|
||||
|
||||
# Encrypted JSON for provider-specific auth
|
||||
credentials = models.BinaryField(
|
||||
blank=False, null=False, help_text="Encrypted JSON credentials for the provider"
|
||||
)
|
||||
|
||||
is_active = models.BooleanField(default=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.get_provider_type_display()} ({self.tenant_id})"
|
||||
|
||||
def clean(self):
|
||||
super().clean()
|
||||
|
||||
@property
|
||||
def credentials_decoded(self):
|
||||
if not self.credentials:
|
||||
return None
|
||||
try:
|
||||
decrypted_data = fernet.decrypt(bytes(self.credentials))
|
||||
return json.loads(decrypted_data.decode())
|
||||
except (InvalidToken, json.JSONDecodeError) as e:
|
||||
logger.warning("Failed to decrypt provider credentials: %s", e)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Unexpected error while decrypting provider credentials: %s", e
|
||||
)
|
||||
return None
|
||||
|
||||
@credentials_decoded.setter
|
||||
def credentials_decoded(self, value):
|
||||
"""
|
||||
Set and encrypt credentials (assumes serializer performed validation).
|
||||
"""
|
||||
if not value:
|
||||
raise ModelValidationError(
|
||||
detail="Credentials are required",
|
||||
code="invalid_credentials",
|
||||
pointer="/data/attributes/credentials",
|
||||
)
|
||||
self.credentials = fernet.encrypt(json.dumps(value).encode())
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "lighthouse_provider_configurations"
|
||||
|
||||
constraints = [
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=["tenant_id", "provider_type"],
|
||||
name="unique_provider_config_per_tenant",
|
||||
),
|
||||
]
|
||||
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_type"],
|
||||
name="lh_pc_tenant_type_idx",
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "lighthouse-providers"
|
||||
|
||||
|
||||
class LighthouseTenantConfiguration(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Tenant-level Lighthouse settings (business context and defaults).
|
||||
One record per tenant.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
|
||||
business_context = models.TextField(blank=True, default="")
|
||||
|
||||
# Preferred provider key (e.g., "openai", "bedrock", "openai_compatible")
|
||||
default_provider = models.CharField(max_length=50, blank=True)
|
||||
|
||||
# Mapping of provider -> model id, e.g., {"openai": "gpt-4o", "bedrock": "anthropic.claude-v2"}
|
||||
default_models = models.JSONField(default=dict, blank=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"Lighthouse Tenant Config for {self.tenant_id}"
|
||||
|
||||
def clean(self):
|
||||
super().clean()
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "lighthouse_tenant_config"
|
||||
|
||||
constraints = [
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=["tenant_id"], name="unique_tenant_lighthouse_config"
|
||||
),
|
||||
]
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "lighthouse-config"
|
||||
|
||||
|
||||
class LighthouseProviderModels(RowLevelSecurityProtectedModel):
|
||||
"""
|
||||
Per-tenant, per-provider configuration list of available LLM models.
|
||||
RLS-protected; populated via provider API using tenant-scoped credentials.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid4, editable=False)
|
||||
inserted_at = models.DateTimeField(auto_now_add=True, editable=False)
|
||||
updated_at = models.DateTimeField(auto_now=True, editable=False)
|
||||
|
||||
# Scope to a specific provider configuration within a tenant
|
||||
provider_configuration = models.ForeignKey(
|
||||
LighthouseProviderConfiguration,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="available_models",
|
||||
)
|
||||
model_id = models.CharField(max_length=100)
|
||||
|
||||
# Human-friendly model name
|
||||
model_name = models.CharField(max_length=100)
|
||||
|
||||
# Model-specific default parameters (e.g., temperature, max_tokens)
|
||||
default_parameters = models.JSONField(default=dict, blank=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.provider_configuration.provider_type}:{self.model_id} ({self.tenant_id})"
|
||||
|
||||
class Meta(RowLevelSecurityProtectedModel.Meta):
|
||||
db_table = "lighthouse_provider_models"
|
||||
constraints = [
|
||||
RowLevelSecurityConstraint(
|
||||
field="tenant_id",
|
||||
name="rls_on_%(class)s",
|
||||
statements=["SELECT", "INSERT", "UPDATE", "DELETE"],
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=["tenant_id", "provider_configuration", "model_id"],
|
||||
name="unique_provider_model_per_configuration",
|
||||
),
|
||||
]
|
||||
indexes = [
|
||||
models.Index(
|
||||
fields=["tenant_id", "provider_configuration"],
|
||||
name="lh_prov_models_cfg_idx",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -6,7 +6,14 @@ from django.dispatch import receiver
|
||||
from django_celery_results.backends.database import DatabaseBackend
|
||||
|
||||
from api.db_utils import delete_related_daily_task
|
||||
from api.models import Membership, Provider, TenantAPIKey, User
|
||||
from api.models import (
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseTenantConfiguration,
|
||||
Membership,
|
||||
Provider,
|
||||
TenantAPIKey,
|
||||
User,
|
||||
)
|
||||
|
||||
|
||||
def create_task_result_on_publish(sender=None, headers=None, **kwargs): # noqa: F841
|
||||
@@ -56,3 +63,33 @@ def revoke_membership_api_keys(sender, instance, **kwargs): # noqa: F841
|
||||
TenantAPIKey.objects.filter(
|
||||
entity=instance.user, tenant_id=instance.tenant.id
|
||||
).update(revoked=True)
|
||||
|
||||
|
||||
@receiver(pre_delete, sender=LighthouseProviderConfiguration)
|
||||
def cleanup_lighthouse_defaults_before_delete(sender, instance, **kwargs): # noqa: F841
|
||||
"""
|
||||
Ensure tenant Lighthouse defaults do not reference a soon-to-be-deleted provider.
|
||||
|
||||
This runs for both per-instance deletes and queryset (bulk) deletes.
|
||||
"""
|
||||
try:
|
||||
tenant_cfg = LighthouseTenantConfiguration.objects.get(
|
||||
tenant_id=instance.tenant_id
|
||||
)
|
||||
except LighthouseTenantConfiguration.DoesNotExist:
|
||||
return
|
||||
|
||||
updated = False
|
||||
defaults = tenant_cfg.default_models or {}
|
||||
|
||||
if instance.provider_type in defaults:
|
||||
defaults.pop(instance.provider_type, None)
|
||||
tenant_cfg.default_models = defaults
|
||||
updated = True
|
||||
|
||||
if tenant_cfg.default_provider == instance.provider_type:
|
||||
tenant_cfg.default_provider = ""
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
tenant_cfg.save()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -35,6 +35,9 @@ from api.db_router import MainRouter
|
||||
from api.models import (
|
||||
Integration,
|
||||
Invitation,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
LighthouseTenantConfiguration,
|
||||
Membership,
|
||||
Processor,
|
||||
Provider,
|
||||
@@ -8703,3 +8706,678 @@ class TestTenantApiKeyViewSet:
|
||||
# Verify error object structure
|
||||
error = response_data["errors"][0]
|
||||
assert "detail" in error or "title" in error
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestLighthouseTenantConfigViewSet:
|
||||
"""Test Lighthouse tenant configuration endpoint (singleton pattern)"""
|
||||
|
||||
def test_lighthouse_tenant_config_create_via_patch(self, authenticated_client):
|
||||
"""Test creating a tenant config successfully via PATCH (upsert)"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-config",
|
||||
"attributes": {
|
||||
"business_context": "Test business context for security analysis",
|
||||
"default_provider": "",
|
||||
"default_models": {},
|
||||
},
|
||||
}
|
||||
}
|
||||
response = authenticated_client.patch(
|
||||
reverse("lighthouse-config"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert (
|
||||
data["attributes"]["business_context"]
|
||||
== "Test business context for security analysis"
|
||||
)
|
||||
assert data["attributes"]["default_provider"] == ""
|
||||
assert data["attributes"]["default_models"] == {}
|
||||
|
||||
def test_lighthouse_tenant_config_upsert_behavior(self, authenticated_client):
|
||||
"""Test that PATCH creates config if not exists and updates if exists (upsert)"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-config",
|
||||
"attributes": {
|
||||
"business_context": "First config",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
# First PATCH creates the config
|
||||
response = authenticated_client.patch(
|
||||
reverse("lighthouse-config"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
first_data = response.json()["data"]
|
||||
assert first_data["attributes"]["business_context"] == "First config"
|
||||
|
||||
# Second PATCH updates the same config (not creating a duplicate)
|
||||
payload["data"]["attributes"]["business_context"] = "Updated config"
|
||||
response = authenticated_client.patch(
|
||||
reverse("lighthouse-config"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
second_data = response.json()["data"]
|
||||
assert second_data["attributes"]["business_context"] == "Updated config"
|
||||
# Verify it's the same config (same ID)
|
||||
assert first_data["id"] == second_data["id"]
|
||||
|
||||
@patch("openai.OpenAI")
|
||||
def test_lighthouse_tenant_config_retrieve(
|
||||
self, mock_openai_client, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Test retrieving the singleton tenant config with proper provider and model validation"""
|
||||
|
||||
# Mock OpenAI client and models response
|
||||
mock_models_response = Mock()
|
||||
mock_models_response.data = [
|
||||
Mock(id="gpt-4o"),
|
||||
Mock(id="gpt-4o-mini"),
|
||||
Mock(id="gpt-5"),
|
||||
]
|
||||
mock_openai_client.return_value.models.list.return_value = mock_models_response
|
||||
|
||||
# Create OpenAI provider configuration
|
||||
provider_config = LighthouseProviderConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_type="openai",
|
||||
credentials=b'{"api_key": "sk-test1234567890T3BlbkFJtest1234567890"}',
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
# Create provider models (simulating refresh)
|
||||
LighthouseProviderModels.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_configuration=provider_config,
|
||||
model_id="gpt-4o",
|
||||
default_parameters={},
|
||||
)
|
||||
LighthouseProviderModels.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
provider_configuration=provider_config,
|
||||
model_id="gpt-4o-mini",
|
||||
default_parameters={},
|
||||
)
|
||||
|
||||
# Create tenant configuration with valid provider and model
|
||||
config = LighthouseTenantConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
business_context="Test context",
|
||||
default_provider="openai",
|
||||
default_models={"openai": "gpt-4o"},
|
||||
)
|
||||
|
||||
# Retrieve and verify the configuration
|
||||
response = authenticated_client.get(reverse("lighthouse-config"))
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
data = response.json()["data"]
|
||||
assert data["id"] == str(config.id)
|
||||
assert data["attributes"]["business_context"] == "Test context"
|
||||
assert data["attributes"]["default_provider"] == "openai"
|
||||
assert data["attributes"]["default_models"] == {"openai": "gpt-4o"}
|
||||
|
||||
def test_lighthouse_tenant_config_retrieve_not_found(self, authenticated_client):
|
||||
"""Test GET when config doesn't exist returns 404"""
|
||||
response = authenticated_client.get(reverse("lighthouse-config"))
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
assert "not found" in response.json()["errors"][0]["detail"].lower()
|
||||
|
||||
def test_lighthouse_tenant_config_partial_update(
|
||||
self, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Test updating tenant config fields"""
|
||||
from api.models import LighthouseTenantConfiguration
|
||||
|
||||
# Create config first
|
||||
config = LighthouseTenantConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
business_context="Original context",
|
||||
default_provider="",
|
||||
default_models={},
|
||||
)
|
||||
|
||||
# Update it
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-config",
|
||||
"attributes": {
|
||||
"business_context": "Updated context for cloud security",
|
||||
},
|
||||
}
|
||||
}
|
||||
response = authenticated_client.patch(
|
||||
reverse("lighthouse-config"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
|
||||
# Verify update
|
||||
config.refresh_from_db()
|
||||
assert config.business_context == "Updated context for cloud security"
|
||||
|
||||
def test_lighthouse_tenant_config_update_invalid_provider(
|
||||
self, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Test validation fails when default_provider is not configured and active"""
|
||||
from api.models import LighthouseTenantConfiguration
|
||||
|
||||
# Create config first
|
||||
LighthouseTenantConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
business_context="Test",
|
||||
)
|
||||
|
||||
# Try to set invalid provider
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-config",
|
||||
"attributes": {
|
||||
"default_provider": "nonexistent-provider",
|
||||
},
|
||||
}
|
||||
}
|
||||
response = authenticated_client.patch(
|
||||
reverse("lighthouse-config"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "provider" in response.json()["errors"][0]["detail"].lower()
|
||||
|
||||
def test_lighthouse_tenant_config_update_invalid_json_format(
|
||||
self, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Test that invalid JSON payload is rejected"""
|
||||
from api.models import LighthouseTenantConfiguration
|
||||
|
||||
# Create config first
|
||||
LighthouseTenantConfiguration.objects.create(
|
||||
tenant_id=tenants_fixture[0].id,
|
||||
business_context="Test",
|
||||
)
|
||||
|
||||
# Send invalid JSON
|
||||
response = authenticated_client.patch(
|
||||
reverse("lighthouse-config"),
|
||||
data="invalid json",
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestLighthouseProviderConfigViewSet:
|
||||
"""Tests for LighthouseProviderConfiguration create validations"""
|
||||
|
||||
def test_invalid_provider_type(self, authenticated_client):
|
||||
"""Add invalid provider (testprovider) should error"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "testprovider",
|
||||
"credentials": {"api_key": "sk-testT3BlbkFJkey"},
|
||||
},
|
||||
}
|
||||
}
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_openai_missing_credentials(self, authenticated_client):
|
||||
"""OpenAI provider without credentials should error"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
},
|
||||
}
|
||||
}
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"credentials",
|
||||
[
|
||||
{}, # empty credentials
|
||||
{"token": "sk-testT3BlbkFJkey"}, # wrong key name
|
||||
{"api_key": "ks-invalid-format"}, # wrong format
|
||||
],
|
||||
)
|
||||
def test_openai_invalid_credentials(self, authenticated_client, credentials):
|
||||
"""OpenAI provider with invalid credentials should error"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
"credentials": credentials,
|
||||
},
|
||||
}
|
||||
}
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_openai_valid_credentials_success(self, authenticated_client):
|
||||
"""OpenAI provider with valid sk-xxx format should succeed"""
|
||||
valid_key = "sk-abc123T3BlbkFJxyz456"
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
"credentials": {"api_key": valid_key},
|
||||
},
|
||||
}
|
||||
}
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp.status_code == status.HTTP_201_CREATED
|
||||
data = resp.json()["data"]
|
||||
|
||||
masked_creds = data["attributes"].get("credentials")
|
||||
assert masked_creds is not None
|
||||
assert "api_key" in masked_creds
|
||||
assert masked_creds["api_key"] == ("*" * len(valid_key))
|
||||
|
||||
def test_openai_provider_duplicate_per_tenant(self, authenticated_client):
|
||||
"""If an OpenAI provider exists for tenant, creating again should error"""
|
||||
valid_key = "sk-dup123T3BlbkFJdup456"
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
"credentials": {"api_key": valid_key},
|
||||
},
|
||||
}
|
||||
}
|
||||
# First creation succeeds
|
||||
resp1 = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp1.status_code == status.HTTP_201_CREATED
|
||||
|
||||
# Second creation should fail with validation error
|
||||
resp2 = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp2.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "already exists" in str(resp2.json()).lower()
|
||||
|
||||
def test_openai_patch_base_url_and_is_active(self, authenticated_client):
|
||||
"""After creating, should be able to patch base_url and is_active"""
|
||||
valid_key = "sk-patch123T3BlbkFJpatch456"
|
||||
create_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
"credentials": {"api_key": valid_key},
|
||||
},
|
||||
}
|
||||
}
|
||||
create_resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=create_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert create_resp.status_code == status.HTTP_201_CREATED
|
||||
provider_id = create_resp.json()["data"]["id"]
|
||||
|
||||
patch_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"id": provider_id,
|
||||
"attributes": {
|
||||
"base_url": "https://api.example.com/v1",
|
||||
"is_active": False,
|
||||
},
|
||||
}
|
||||
}
|
||||
patch_resp = authenticated_client.patch(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}),
|
||||
data=patch_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert patch_resp.status_code == status.HTTP_200_OK
|
||||
updated = patch_resp.json()["data"]["attributes"]
|
||||
assert updated["base_url"] == "https://api.example.com/v1"
|
||||
assert updated["is_active"] is False
|
||||
|
||||
def test_openai_patch_invalid_credentials(self, authenticated_client):
|
||||
"""PATCH with invalid credentials.api_key should error (400)"""
|
||||
valid_key = "sk-ok123T3BlbkFJok456"
|
||||
create_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
"credentials": {"api_key": valid_key},
|
||||
},
|
||||
}
|
||||
}
|
||||
create_resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=create_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert create_resp.status_code == status.HTTP_201_CREATED
|
||||
provider_id = create_resp.json()["data"]["id"]
|
||||
|
||||
# Try patch with invalid api_key format
|
||||
patch_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"id": provider_id,
|
||||
"attributes": {
|
||||
"credentials": {"api_key": "ks-invalid-format"},
|
||||
},
|
||||
}
|
||||
}
|
||||
patch_resp = authenticated_client.patch(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}),
|
||||
data=patch_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert patch_resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_openai_get_masking_and_fields_filter(self, authenticated_client):
|
||||
valid_key = "sk-get123T3BlbkFJget456"
|
||||
create_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "openai",
|
||||
"credentials": {"api_key": valid_key},
|
||||
},
|
||||
}
|
||||
}
|
||||
create_resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=create_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert create_resp.status_code == status.HTTP_201_CREATED
|
||||
provider_id = create_resp.json()["data"]["id"]
|
||||
|
||||
# Default GET should return masked credentials
|
||||
get_resp = authenticated_client.get(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id})
|
||||
)
|
||||
assert get_resp.status_code == status.HTTP_200_OK
|
||||
masked = get_resp.json()["data"]["attributes"]["credentials"]["api_key"]
|
||||
assert masked == ("*" * len(valid_key))
|
||||
|
||||
# Fields filter should return decrypted credentials structure
|
||||
get_full = authenticated_client.get(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id})
|
||||
+ "?fields[lighthouse-providers]=credentials"
|
||||
)
|
||||
assert get_full.status_code == status.HTTP_200_OK
|
||||
creds = get_full.json()["data"]["attributes"]["credentials"]
|
||||
assert creds["api_key"] == valid_key
|
||||
|
||||
def test_delete_provider_updates_tenant_defaults(
|
||||
self, authenticated_client, tenants_fixture
|
||||
):
|
||||
"""Deleting a provider config should clear tenant default_provider and its default_model entry."""
|
||||
|
||||
tenant = tenants_fixture[0]
|
||||
|
||||
# Create provider configuration to delete
|
||||
provider = LighthouseProviderConfiguration.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
provider_type="openai",
|
||||
credentials=b'{"api_key":"sk-test123T3BlbkFJ"}',
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
# Seed tenant defaults referencing the provider we will delete
|
||||
cfg = LighthouseTenantConfiguration.objects.create(
|
||||
tenant_id=tenant.id,
|
||||
business_context="Test",
|
||||
default_provider="openai",
|
||||
default_models={"openai": "gpt-4o", "other": "model-x"},
|
||||
)
|
||||
|
||||
# Delete via API and validate response
|
||||
url = reverse("lighthouse-providers-detail", kwargs={"pk": str(provider.id)})
|
||||
resp = authenticated_client.delete(url)
|
||||
assert resp.status_code in (
|
||||
status.HTTP_204_NO_CONTENT,
|
||||
status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
# Tenant defaults should be updated
|
||||
cfg.refresh_from_db()
|
||||
assert cfg.default_provider == ""
|
||||
assert "openai" not in cfg.default_models
|
||||
|
||||
# Unrelated entries should remain untouched
|
||||
assert cfg.default_models.get("other") == "model-x"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"credentials",
|
||||
[
|
||||
{}, # empty credentials
|
||||
{
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE"
|
||||
}, # missing secret_access_key and region
|
||||
{
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}, # missing access_key_id and region
|
||||
{
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
}, # missing region
|
||||
{ # invalid access_key_id format (not starting with AKIA)
|
||||
"access_key_id": "ABCD0123456789ABCDEF",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-east-1",
|
||||
},
|
||||
{ # invalid access_key_id format (wrong length)
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPL",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-east-1",
|
||||
},
|
||||
{ # invalid secret_access_key format (wrong length)
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEK",
|
||||
"region": "us-east-1",
|
||||
},
|
||||
{ # invalid region format
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "invalid-region",
|
||||
},
|
||||
{ # invalid region format (uppercase)
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "US-EAST-1",
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_bedrock_invalid_credentials(self, authenticated_client, credentials):
|
||||
"""Bedrock provider with invalid credentials should error"""
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "bedrock",
|
||||
"credentials": credentials,
|
||||
},
|
||||
}
|
||||
}
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
def test_bedrock_valid_credentials_success(self, authenticated_client):
|
||||
"""Bedrock provider with valid AWS credentials should succeed and mask credentials"""
|
||||
valid_credentials = {
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-east-1",
|
||||
}
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "bedrock",
|
||||
"credentials": valid_credentials,
|
||||
},
|
||||
}
|
||||
}
|
||||
resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp.status_code == status.HTTP_201_CREATED
|
||||
data = resp.json()["data"]
|
||||
|
||||
# Verify credentials are returned masked
|
||||
masked_creds = data["attributes"].get("credentials")
|
||||
assert masked_creds is not None
|
||||
assert "access_key_id" in masked_creds
|
||||
assert "secret_access_key" in masked_creds
|
||||
assert "region" in masked_creds
|
||||
# Verify all characters are masked with asterisks
|
||||
assert all(c == "*" for c in masked_creds["access_key_id"])
|
||||
assert all(c == "*" for c in masked_creds["secret_access_key"])
|
||||
|
||||
def test_bedrock_provider_duplicate_per_tenant(self, authenticated_client):
|
||||
"""Creating a second Bedrock provider for same tenant should fail"""
|
||||
valid_credentials = {
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "us-west-2",
|
||||
}
|
||||
payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "bedrock",
|
||||
"credentials": valid_credentials,
|
||||
},
|
||||
}
|
||||
}
|
||||
# First creation succeeds
|
||||
resp1 = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp1.status_code == status.HTTP_201_CREATED
|
||||
|
||||
# Second creation should fail with validation error
|
||||
resp2 = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert resp2.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "already exists" in str(resp2.json()).lower()
|
||||
|
||||
def test_bedrock_patch_credentials_and_fields_filter(self, authenticated_client):
|
||||
"""PATCH credentials and verify fields filter returns decrypted values"""
|
||||
valid_credentials = {
|
||||
"access_key_id": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"region": "eu-west-1",
|
||||
}
|
||||
create_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"attributes": {
|
||||
"provider_type": "bedrock",
|
||||
"credentials": valid_credentials,
|
||||
},
|
||||
}
|
||||
}
|
||||
create_resp = authenticated_client.post(
|
||||
reverse("lighthouse-providers-list"),
|
||||
data=create_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert create_resp.status_code == status.HTTP_201_CREATED
|
||||
provider_id = create_resp.json()["data"]["id"]
|
||||
|
||||
# Update credentials with new valid ones
|
||||
new_credentials = {
|
||||
"access_key_id": "AKIAZZZZZZZZZZZZZZZZ",
|
||||
"secret_access_key": "aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789+/==",
|
||||
"region": "ap-south-1",
|
||||
}
|
||||
patch_payload = {
|
||||
"data": {
|
||||
"type": "lighthouse-providers",
|
||||
"id": provider_id,
|
||||
"attributes": {
|
||||
"credentials": new_credentials,
|
||||
"is_active": False,
|
||||
},
|
||||
}
|
||||
}
|
||||
patch_resp = authenticated_client.patch(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id}),
|
||||
data=patch_payload,
|
||||
content_type=API_JSON_CONTENT_TYPE,
|
||||
)
|
||||
assert patch_resp.status_code == status.HTTP_200_OK
|
||||
updated = patch_resp.json()["data"]["attributes"]
|
||||
assert updated["is_active"] is False
|
||||
|
||||
# Default GET should return masked credentials
|
||||
get_resp = authenticated_client.get(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id})
|
||||
)
|
||||
assert get_resp.status_code == status.HTTP_200_OK
|
||||
masked = get_resp.json()["data"]["attributes"]["credentials"]
|
||||
assert all(c == "*" for c in masked["access_key_id"])
|
||||
assert all(c == "*" for c in masked["secret_access_key"])
|
||||
|
||||
# Fields filter should return decrypted credentials
|
||||
get_full = authenticated_client.get(
|
||||
reverse("lighthouse-providers-detail", kwargs={"pk": provider_id})
|
||||
+ "?fields[lighthouse-providers]=credentials"
|
||||
)
|
||||
assert get_full.status_code == status.HTTP_200_OK
|
||||
creds = get_full.json()["data"]["attributes"]["credentials"]
|
||||
assert creds["access_key_id"] == new_credentials["access_key_id"]
|
||||
assert creds["secret_access_key"] == new_credentials["secret_access_key"]
|
||||
assert creds["region"] == new_credentials["region"]
|
||||
|
||||
52
api/src/backend/api/v1/serializer_utils/lighthouse.py
Normal file
52
api/src/backend/api/v1/serializer_utils/lighthouse.py
Normal file
@@ -0,0 +1,52 @@
|
||||
import re
|
||||
|
||||
from rest_framework_json_api import serializers
|
||||
|
||||
|
||||
class OpenAICredentialsSerializer(serializers.Serializer):
|
||||
api_key = serializers.CharField()
|
||||
|
||||
def validate_api_key(self, value: str) -> str:
|
||||
pattern = r"^sk-[\w-]+$"
|
||||
if not re.match(pattern, value or ""):
|
||||
raise serializers.ValidationError("Invalid OpenAI API key format.")
|
||||
return value
|
||||
|
||||
|
||||
class BedrockCredentialsSerializer(serializers.Serializer):
|
||||
"""
|
||||
Serializer for AWS Bedrock credentials validation.
|
||||
|
||||
Validates long-term AWS credentials (AKIA) and region format.
|
||||
"""
|
||||
|
||||
access_key_id = serializers.CharField()
|
||||
secret_access_key = serializers.CharField()
|
||||
region = serializers.CharField()
|
||||
|
||||
def validate_access_key_id(self, value: str) -> str:
|
||||
"""Validate AWS access key ID format (AKIA for long-term credentials)."""
|
||||
pattern = r"^AKIA[0-9A-Z]{16}$"
|
||||
if not re.match(pattern, value or ""):
|
||||
raise serializers.ValidationError(
|
||||
"Invalid AWS access key ID format. Must be AKIA followed by 16 alphanumeric characters."
|
||||
)
|
||||
return value
|
||||
|
||||
def validate_secret_access_key(self, value: str) -> str:
|
||||
"""Validate AWS secret access key format (40 base64 characters)."""
|
||||
pattern = r"^[A-Za-z0-9/+=]{40}$"
|
||||
if not re.match(pattern, value or ""):
|
||||
raise serializers.ValidationError(
|
||||
"Invalid AWS secret access key format. Must be 40 base64 characters."
|
||||
)
|
||||
return value
|
||||
|
||||
def validate_region(self, value: str) -> str:
|
||||
"""Validate AWS region format."""
|
||||
pattern = r"^[a-z]{2}-[a-z]+-\d+$"
|
||||
if not re.match(pattern, value or ""):
|
||||
raise serializers.ValidationError(
|
||||
"Invalid AWS region format. Expected format like 'us-east-1' or 'eu-west-2'."
|
||||
)
|
||||
return value
|
||||
@@ -6,8 +6,10 @@ from django.conf import settings
|
||||
from django.contrib.auth import authenticate
|
||||
from django.contrib.auth.models import update_last_login
|
||||
from django.contrib.auth.password_validation import validate_password
|
||||
from django.db import IntegrityError
|
||||
from drf_spectacular.utils import extend_schema_field
|
||||
from jwt.exceptions import InvalidKeyError
|
||||
from rest_framework.reverse import reverse
|
||||
from rest_framework.validators import UniqueTogetherValidator
|
||||
from rest_framework_json_api import serializers
|
||||
from rest_framework_json_api.relations import SerializerMethodResourceRelatedField
|
||||
@@ -25,6 +27,9 @@ from api.models import (
|
||||
Invitation,
|
||||
InvitationRoleRelationship,
|
||||
LighthouseConfiguration,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
LighthouseTenantConfiguration,
|
||||
Membership,
|
||||
Processor,
|
||||
Provider,
|
||||
@@ -54,6 +59,10 @@ from api.v1.serializer_utils.integrations import (
|
||||
S3ConfigSerializer,
|
||||
SecurityHubConfigSerializer,
|
||||
)
|
||||
from api.v1.serializer_utils.lighthouse import (
|
||||
BedrockCredentialsSerializer,
|
||||
OpenAICredentialsSerializer,
|
||||
)
|
||||
from api.v1.serializer_utils.processors import ProcessorConfigField
|
||||
from api.v1.serializer_utils.providers import ProviderSecretField
|
||||
from prowler.lib.mutelist.mutelist import Mutelist
|
||||
@@ -2750,6 +2759,16 @@ class LighthouseConfigCreateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
"updated_at": {"read_only": True},
|
||||
}
|
||||
|
||||
def validate_temperature(self, value):
|
||||
if not 0 <= value <= 1:
|
||||
raise ValidationError("Temperature must be between 0 and 1.")
|
||||
return value
|
||||
|
||||
def validate_max_tokens(self, value):
|
||||
if not 500 <= value <= 5000:
|
||||
raise ValidationError("Max tokens must be between 500 and 5000.")
|
||||
return value
|
||||
|
||||
def validate(self, attrs):
|
||||
tenant_id = self.context.get("request").tenant_id
|
||||
if LighthouseConfiguration.objects.filter(tenant_id=tenant_id).exists():
|
||||
@@ -2758,6 +2777,11 @@ class LighthouseConfigCreateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
"tenant_id": "Lighthouse configuration already exists for this tenant."
|
||||
}
|
||||
)
|
||||
api_key = attrs.get("api_key")
|
||||
if api_key is not None:
|
||||
OpenAICredentialsSerializer(data={"api_key": api_key}).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
return super().validate(attrs)
|
||||
|
||||
def create(self, validated_data):
|
||||
@@ -2802,6 +2826,24 @@ class LighthouseConfigUpdateSerializer(BaseWriteSerializer):
|
||||
"max_tokens": {"required": False},
|
||||
}
|
||||
|
||||
def validate_temperature(self, value):
|
||||
if not 0 <= value <= 1:
|
||||
raise ValidationError("Temperature must be between 0 and 1.")
|
||||
return value
|
||||
|
||||
def validate_max_tokens(self, value):
|
||||
if not 500 <= value <= 5000:
|
||||
raise ValidationError("Max tokens must be between 500 and 5000.")
|
||||
return value
|
||||
|
||||
def validate(self, attrs):
|
||||
api_key = attrs.get("api_key", None)
|
||||
if api_key is not None:
|
||||
OpenAICredentialsSerializer(data={"api_key": api_key}).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
return super().validate(attrs)
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
api_key = validated_data.pop("api_key", None)
|
||||
instance = super().update(instance, validated_data)
|
||||
@@ -2931,3 +2973,380 @@ class TenantApiKeyUpdateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
):
|
||||
raise ValidationError("An API key with this name already exists.")
|
||||
return value
|
||||
|
||||
|
||||
# Lighthouse: Provider configurations
|
||||
|
||||
|
||||
class LighthouseProviderConfigSerializer(RLSSerializer):
|
||||
"""
|
||||
Read serializer for LighthouseProviderConfiguration.
|
||||
"""
|
||||
|
||||
# Decrypted credentials are only returned in to_representation when requested
|
||||
credentials = serializers.JSONField(required=False, read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderConfiguration
|
||||
fields = [
|
||||
"id",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
"provider_type",
|
||||
"base_url",
|
||||
"is_active",
|
||||
"credentials",
|
||||
"url",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
"inserted_at": {"read_only": True},
|
||||
"updated_at": {"read_only": True},
|
||||
"is_active": {"read_only": True},
|
||||
"url": {"read_only": True, "view_name": "lighthouse-providers-detail"},
|
||||
}
|
||||
|
||||
class JSONAPIMeta:
|
||||
resource_name = "lighthouse-providers"
|
||||
|
||||
def to_representation(self, instance):
|
||||
data = super().to_representation(instance)
|
||||
# Support JSON:API fields filter: fields[lighthouse-providers]=credentials,base_url
|
||||
fields_param = self.context.get("request", None) and self.context[
|
||||
"request"
|
||||
].query_params.get("fields[lighthouse-providers]", "")
|
||||
|
||||
creds = instance.credentials_decoded
|
||||
|
||||
requested_fields = (
|
||||
[f.strip() for f in fields_param.split(",")] if fields_param else []
|
||||
)
|
||||
|
||||
if "credentials" in requested_fields:
|
||||
# Return full decrypted credentials JSON
|
||||
data["credentials"] = creds
|
||||
else:
|
||||
# Return masked credentials by default
|
||||
def mask_value(value):
|
||||
if isinstance(value, str):
|
||||
return "*" * len(value)
|
||||
if isinstance(value, dict):
|
||||
return {k: mask_value(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [mask_value(v) for v in value]
|
||||
return value
|
||||
|
||||
# Always return masked credentials, even if creds is None
|
||||
if creds is not None:
|
||||
data["credentials"] = mask_value(creds)
|
||||
else:
|
||||
# If credentials_decoded returns None, return None for credentials field
|
||||
data["credentials"] = None
|
||||
|
||||
return data
|
||||
|
||||
|
||||
class LighthouseProviderConfigCreateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
"""
|
||||
Create serializer for LighthouseProviderConfiguration.
|
||||
Accepts credentials as JSON; stored encrypted via credentials_decoded.
|
||||
"""
|
||||
|
||||
credentials = serializers.JSONField(write_only=True, required=True)
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderConfiguration
|
||||
fields = [
|
||||
"provider_type",
|
||||
"base_url",
|
||||
"credentials",
|
||||
"is_active",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"is_active": {"required": False},
|
||||
"base_url": {"required": False, "allow_null": True},
|
||||
}
|
||||
|
||||
def create(self, validated_data):
|
||||
credentials = validated_data.pop("credentials")
|
||||
|
||||
instance = LighthouseProviderConfiguration(**validated_data)
|
||||
instance.tenant_id = self.context.get("tenant_id")
|
||||
instance.credentials_decoded = credentials
|
||||
|
||||
try:
|
||||
instance.save()
|
||||
return instance
|
||||
except IntegrityError:
|
||||
raise ValidationError(
|
||||
{
|
||||
"provider_type": "Configuration for this provider already exists for the tenant."
|
||||
}
|
||||
)
|
||||
|
||||
def validate(self, attrs):
|
||||
provider_type = attrs.get("provider_type")
|
||||
credentials = attrs.get("credentials") or {}
|
||||
|
||||
if provider_type == LighthouseProviderConfiguration.LLMProviderChoices.OPENAI:
|
||||
try:
|
||||
OpenAICredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
elif (
|
||||
provider_type == LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
try:
|
||||
BedrockCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
|
||||
class LighthouseProviderConfigUpdateSerializer(BaseWriteSerializer):
|
||||
"""
|
||||
Update serializer for LighthouseProviderConfiguration.
|
||||
"""
|
||||
|
||||
credentials = serializers.JSONField(write_only=True, required=False)
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderConfiguration
|
||||
fields = [
|
||||
"id",
|
||||
"provider_type",
|
||||
"base_url",
|
||||
"credentials",
|
||||
"is_active",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
"provider_type": {"read_only": True},
|
||||
"base_url": {"required": False, "allow_null": True},
|
||||
"is_active": {"required": False},
|
||||
}
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
credentials = validated_data.pop("credentials", None)
|
||||
|
||||
for attr, value in validated_data.items():
|
||||
setattr(instance, attr, value)
|
||||
|
||||
if credentials is not None:
|
||||
instance.credentials_decoded = credentials
|
||||
|
||||
instance.save()
|
||||
return instance
|
||||
|
||||
def validate(self, attrs):
|
||||
provider_type = getattr(self.instance, "provider_type", None)
|
||||
credentials = attrs.get("credentials", None)
|
||||
|
||||
if (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
|
||||
):
|
||||
try:
|
||||
OpenAICredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
elif (
|
||||
credentials is not None
|
||||
and provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
try:
|
||||
BedrockCredentialsSerializer(data=credentials).is_valid(
|
||||
raise_exception=True
|
||||
)
|
||||
except ValidationError as e:
|
||||
details = e.detail.copy()
|
||||
for key, value in details.items():
|
||||
e.detail[f"credentials/{key}"] = value
|
||||
del e.detail[key]
|
||||
raise e
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
|
||||
# Lighthouse: Tenant configuration
|
||||
|
||||
|
||||
class LighthouseTenantConfigSerializer(RLSSerializer):
|
||||
"""
|
||||
Read serializer for LighthouseTenantConfiguration.
|
||||
"""
|
||||
|
||||
# Build singleton URL without pk
|
||||
url = serializers.SerializerMethodField()
|
||||
|
||||
def get_url(self, obj):
|
||||
request = self.context.get("request")
|
||||
return reverse("lighthouse-config", request=request)
|
||||
|
||||
class Meta:
|
||||
model = LighthouseTenantConfiguration
|
||||
fields = [
|
||||
"id",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
"business_context",
|
||||
"default_provider",
|
||||
"default_models",
|
||||
"url",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
"inserted_at": {"read_only": True},
|
||||
"updated_at": {"read_only": True},
|
||||
"url": {"read_only": True},
|
||||
}
|
||||
|
||||
|
||||
class LighthouseTenantConfigUpdateSerializer(BaseWriteSerializer):
|
||||
class Meta:
|
||||
model = LighthouseTenantConfiguration
|
||||
fields = [
|
||||
"id",
|
||||
"business_context",
|
||||
"default_provider",
|
||||
"default_models",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
}
|
||||
|
||||
def validate(self, attrs):
|
||||
request = self.context.get("request")
|
||||
tenant_id = self.context.get("tenant_id") or (
|
||||
getattr(request, "tenant_id", None) if request else None
|
||||
)
|
||||
|
||||
default_provider = attrs.get(
|
||||
"default_provider", getattr(self.instance, "default_provider", "")
|
||||
)
|
||||
default_models = attrs.get(
|
||||
"default_models", getattr(self.instance, "default_models", {})
|
||||
)
|
||||
|
||||
if default_provider:
|
||||
supported = set(LighthouseProviderConfiguration.LLMProviderChoices.values)
|
||||
if default_provider not in supported:
|
||||
raise ValidationError(
|
||||
{"default_provider": f"Unsupported provider '{default_provider}'."}
|
||||
)
|
||||
if not LighthouseProviderConfiguration.objects.filter(
|
||||
tenant_id=tenant_id, provider_type=default_provider, is_active=True
|
||||
).exists():
|
||||
raise ValidationError(
|
||||
{
|
||||
"default_provider": f"No active configuration found for '{default_provider}'."
|
||||
}
|
||||
)
|
||||
|
||||
if default_models is not None and not isinstance(default_models, dict):
|
||||
raise ValidationError(
|
||||
{"default_models": "Must be an object mapping provider -> model_id."}
|
||||
)
|
||||
|
||||
for provider_type, model_id in (default_models or {}).items():
|
||||
provider_cfg = LighthouseProviderConfiguration.objects.filter(
|
||||
tenant_id=tenant_id, provider_type=provider_type, is_active=True
|
||||
).first()
|
||||
if not provider_cfg:
|
||||
raise ValidationError(
|
||||
{
|
||||
"default_models": f"No active configuration for provider '{provider_type}'."
|
||||
}
|
||||
)
|
||||
if not LighthouseProviderModels.objects.filter(
|
||||
tenant_id=tenant_id,
|
||||
provider_configuration=provider_cfg,
|
||||
model_id=model_id,
|
||||
).exists():
|
||||
raise ValidationError(
|
||||
{
|
||||
"default_models": f"Invalid model '{model_id}' for provider '{provider_type}'."
|
||||
}
|
||||
)
|
||||
|
||||
return super().validate(attrs)
|
||||
|
||||
|
||||
# Lighthouse: Provider models
|
||||
|
||||
|
||||
class LighthouseProviderModelsSerializer(RLSSerializer):
|
||||
"""
|
||||
Read serializer for LighthouseProviderModels.
|
||||
"""
|
||||
|
||||
provider_configuration = serializers.ResourceRelatedField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderModels
|
||||
fields = [
|
||||
"id",
|
||||
"inserted_at",
|
||||
"updated_at",
|
||||
"provider_configuration",
|
||||
"model_id",
|
||||
"model_name",
|
||||
"default_parameters",
|
||||
"url",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
"inserted_at": {"read_only": True},
|
||||
"updated_at": {"read_only": True},
|
||||
"url": {"read_only": True, "view_name": "lighthouse-models-detail"},
|
||||
}
|
||||
|
||||
|
||||
class LighthouseProviderModelsCreateSerializer(RLSSerializer, BaseWriteSerializer):
|
||||
provider_configuration = serializers.ResourceRelatedField(
|
||||
queryset=LighthouseProviderConfiguration.objects.all()
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = LighthouseProviderModels
|
||||
fields = [
|
||||
"provider_configuration",
|
||||
"model_id",
|
||||
"default_parameters",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"default_parameters": {"required": False},
|
||||
}
|
||||
|
||||
|
||||
class LighthouseProviderModelsUpdateSerializer(BaseWriteSerializer):
|
||||
class Meta:
|
||||
model = LighthouseProviderModels
|
||||
fields = [
|
||||
"id",
|
||||
"default_parameters",
|
||||
]
|
||||
extra_kwargs = {
|
||||
"id": {"read_only": True},
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@ from api.v1.views import (
|
||||
InvitationAcceptViewSet,
|
||||
InvitationViewSet,
|
||||
LighthouseConfigViewSet,
|
||||
LighthouseProviderConfigViewSet,
|
||||
LighthouseProviderModelsViewSet,
|
||||
LighthouseTenantConfigViewSet,
|
||||
MembershipViewSet,
|
||||
OverviewViewSet,
|
||||
ProcessorViewSet,
|
||||
@@ -34,12 +37,12 @@ from api.v1.views import (
|
||||
ScheduleViewSet,
|
||||
SchemaView,
|
||||
TaskViewSet,
|
||||
TenantApiKeyViewSet,
|
||||
TenantFinishACSView,
|
||||
TenantMembersViewSet,
|
||||
TenantViewSet,
|
||||
UserRoleRelationshipView,
|
||||
UserViewSet,
|
||||
TenantApiKeyViewSet,
|
||||
)
|
||||
|
||||
router = routers.DefaultRouter(trailing_slash=False)
|
||||
@@ -67,6 +70,16 @@ router.register(
|
||||
basename="lighthouseconfiguration",
|
||||
)
|
||||
router.register(r"api-keys", TenantApiKeyViewSet, basename="api-key")
|
||||
router.register(
|
||||
r"lighthouse/providers",
|
||||
LighthouseProviderConfigViewSet,
|
||||
basename="lighthouse-providers",
|
||||
)
|
||||
router.register(
|
||||
r"lighthouse/models",
|
||||
LighthouseProviderModelsViewSet,
|
||||
basename="lighthouse-models",
|
||||
)
|
||||
|
||||
tenants_router = routers.NestedSimpleRouter(router, r"tenants", lookup="tenant")
|
||||
tenants_router.register(
|
||||
@@ -137,6 +150,14 @@ urlpatterns = [
|
||||
),
|
||||
name="provider_group-providers-relationship",
|
||||
),
|
||||
# Lighthouse tenant config as singleton endpoint
|
||||
path(
|
||||
"lighthouse/configuration",
|
||||
LighthouseTenantConfigViewSet.as_view(
|
||||
{"get": "list", "patch": "partial_update"}
|
||||
),
|
||||
name="lighthouse-config",
|
||||
),
|
||||
# API endpoint to start SAML SSO flow
|
||||
path(
|
||||
"auth/saml/initiate/", SAMLInitiateAPIView.as_view(), name="api_saml_initiate"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fnmatch
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -60,11 +61,13 @@ from tasks.tasks import (
|
||||
backfill_scan_resource_summaries_task,
|
||||
check_integration_connection_task,
|
||||
check_lighthouse_connection_task,
|
||||
check_lighthouse_provider_connection_task,
|
||||
check_provider_connection_task,
|
||||
delete_provider_task,
|
||||
delete_tenant_task,
|
||||
jira_integration_task,
|
||||
perform_scan_task,
|
||||
refresh_lighthouse_provider_models_task,
|
||||
)
|
||||
|
||||
from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset
|
||||
@@ -84,6 +87,8 @@ from api.filters import (
|
||||
InvitationFilter,
|
||||
LatestFindingFilter,
|
||||
LatestResourceFilter,
|
||||
LighthouseProviderConfigFilter,
|
||||
LighthouseProviderModelsFilter,
|
||||
MembershipFilter,
|
||||
ProcessorFilter,
|
||||
ProviderFilter,
|
||||
@@ -106,6 +111,9 @@ from api.models import (
|
||||
Integration,
|
||||
Invitation,
|
||||
LighthouseConfiguration,
|
||||
LighthouseProviderConfiguration,
|
||||
LighthouseProviderModels,
|
||||
LighthouseTenantConfiguration,
|
||||
Membership,
|
||||
Processor,
|
||||
Provider,
|
||||
@@ -160,6 +168,12 @@ from api.v1.serializers import (
|
||||
LighthouseConfigCreateSerializer,
|
||||
LighthouseConfigSerializer,
|
||||
LighthouseConfigUpdateSerializer,
|
||||
LighthouseProviderConfigCreateSerializer,
|
||||
LighthouseProviderConfigSerializer,
|
||||
LighthouseProviderConfigUpdateSerializer,
|
||||
LighthouseProviderModelsSerializer,
|
||||
LighthouseTenantConfigSerializer,
|
||||
LighthouseTenantConfigUpdateSerializer,
|
||||
MembershipSerializer,
|
||||
OverviewFindingSerializer,
|
||||
OverviewProviderSerializer,
|
||||
@@ -4177,21 +4191,25 @@ class IntegrationJiraViewSet(BaseRLSViewSet):
|
||||
tags=["Lighthouse AI"],
|
||||
summary="List all Lighthouse AI configurations",
|
||||
description="Retrieve a list of all Lighthouse AI configurations.",
|
||||
deprecated=True,
|
||||
),
|
||||
create=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Create a new Lighthouse AI configuration",
|
||||
description="Create a new Lighthouse AI configuration with the specified details.",
|
||||
deprecated=True,
|
||||
),
|
||||
partial_update=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Partially update a Lighthouse AI configuration",
|
||||
description="Update certain fields of an existing Lighthouse AI configuration.",
|
||||
deprecated=True,
|
||||
),
|
||||
destroy=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Delete a Lighthouse AI configuration",
|
||||
description="Remove a Lighthouse AI configuration by its ID.",
|
||||
deprecated=True,
|
||||
),
|
||||
connection=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
@@ -4199,6 +4217,7 @@ class IntegrationJiraViewSet(BaseRLSViewSet):
|
||||
description="Verify the connection to the OpenAI API for a specific Lighthouse AI configuration.",
|
||||
request=None,
|
||||
responses={202: OpenApiResponse(response=TaskSerializer)},
|
||||
deprecated=True,
|
||||
),
|
||||
)
|
||||
class LighthouseConfigViewSet(BaseRLSViewSet):
|
||||
@@ -4249,6 +4268,253 @@ class LighthouseConfigViewSet(BaseRLSViewSet):
|
||||
)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="List all LLM provider configs",
|
||||
description="Retrieve all LLM provider configurations for the current tenant",
|
||||
),
|
||||
retrieve=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Retrieve LLM provider config",
|
||||
description="Get details for a specific provider configuration in the current tenant.",
|
||||
),
|
||||
create=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Create LLM provider config",
|
||||
description="Create a per-tenant configuration for an LLM provider. Only one configuration per provider type is allowed per tenant.",
|
||||
),
|
||||
partial_update=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Update LLM provider config",
|
||||
description="Partially update a provider configuration (e.g., base_url, is_active).",
|
||||
),
|
||||
destroy=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Delete LLM provider config",
|
||||
description="Delete a provider configuration. Any tenant defaults that reference this provider are cleared during deletion.",
|
||||
),
|
||||
)
|
||||
class LighthouseProviderConfigViewSet(BaseRLSViewSet):
|
||||
queryset = LighthouseProviderConfiguration.objects.all()
|
||||
serializer_class = LighthouseProviderConfigSerializer
|
||||
http_method_names = ["get", "post", "patch", "delete"]
|
||||
filterset_class = LighthouseProviderConfigFilter
|
||||
|
||||
def get_queryset(self):
|
||||
if getattr(self, "swagger_fake_view", False):
|
||||
return LighthouseProviderConfiguration.objects.none()
|
||||
return LighthouseProviderConfiguration.objects.filter(
|
||||
tenant_id=self.request.tenant_id
|
||||
)
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "create":
|
||||
return LighthouseProviderConfigCreateSerializer
|
||||
elif self.action == "partial_update":
|
||||
return LighthouseProviderConfigUpdateSerializer
|
||||
elif self.action in ["connection", "refresh_models"]:
|
||||
return TaskSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
def create(self, request, *args, **kwargs):
|
||||
serializer = self.get_serializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
instance = serializer.save()
|
||||
|
||||
read_serializer = LighthouseProviderConfigSerializer(
|
||||
instance, context=self.get_serializer_context()
|
||||
)
|
||||
headers = self.get_success_headers(read_serializer.data)
|
||||
return Response(
|
||||
data=read_serializer.data,
|
||||
status=status.HTTP_201_CREATED,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
instance = self.get_object()
|
||||
serializer = self.get_serializer(
|
||||
instance,
|
||||
data=request.data,
|
||||
partial=True,
|
||||
context=self.get_serializer_context(),
|
||||
)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
serializer.save()
|
||||
read_serializer = LighthouseProviderConfigSerializer(
|
||||
instance, context=self.get_serializer_context()
|
||||
)
|
||||
return Response(data=read_serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
@extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Check LLM provider connection",
|
||||
description="Validate provider credentials asynchronously and toggle is_active. Supports OpenAI and AWS Bedrock providers.",
|
||||
request=None,
|
||||
responses={202: OpenApiResponse(response=TaskSerializer)},
|
||||
)
|
||||
@action(detail=True, methods=["post"], url_name="connection")
|
||||
def connection(self, request, pk=None):
|
||||
instance = self.get_object()
|
||||
|
||||
with transaction.atomic():
|
||||
task = check_lighthouse_provider_connection_task.delay(
|
||||
provider_config_id=str(instance.id), tenant_id=self.request.tenant_id
|
||||
)
|
||||
|
||||
prowler_task = Task.objects.get(id=task.id)
|
||||
serializer = TaskSerializer(prowler_task)
|
||||
return Response(
|
||||
data=serializer.data,
|
||||
status=status.HTTP_202_ACCEPTED,
|
||||
headers={
|
||||
"Content-Location": reverse(
|
||||
"task-detail", kwargs={"pk": prowler_task.id}
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
@extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Refresh LLM models catalog",
|
||||
description="Fetch available models for this provider configuration and upsert into catalog. Supports OpenAI and AWS Bedrock providers.",
|
||||
request=None,
|
||||
responses={202: OpenApiResponse(response=TaskSerializer)},
|
||||
)
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["post"],
|
||||
url_path="refresh-models",
|
||||
url_name="refresh-models",
|
||||
)
|
||||
def refresh_models(self, request, pk=None):
|
||||
instance = self.get_object()
|
||||
|
||||
with transaction.atomic():
|
||||
task = refresh_lighthouse_provider_models_task.delay(
|
||||
provider_config_id=str(instance.id), tenant_id=self.request.tenant_id
|
||||
)
|
||||
|
||||
prowler_task = Task.objects.get(id=task.id)
|
||||
serializer = TaskSerializer(prowler_task)
|
||||
return Response(
|
||||
data=serializer.data,
|
||||
status=status.HTTP_202_ACCEPTED,
|
||||
headers={
|
||||
"Content-Location": reverse(
|
||||
"task-detail", kwargs={"pk": prowler_task.id}
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Get Lighthouse AI Tenant config",
|
||||
description="Retrieve current tenant-level Lighthouse AI settings. Returns a single configuration object.",
|
||||
),
|
||||
partial_update=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Update Lighthouse AI Tenant config",
|
||||
description="Update tenant-level settings. Validates that the default provider is configured and active and that default model IDs exist for the chosen providers. Auto-creates configuration if it doesn't exist.",
|
||||
),
|
||||
)
|
||||
class LighthouseTenantConfigViewSet(BaseRLSViewSet):
|
||||
"""
|
||||
Singleton endpoint for tenant-level Lighthouse AI configuration.
|
||||
|
||||
This viewset implements a true singleton pattern:
|
||||
- GET returns the single configuration object (or 404 if not found)
|
||||
- PATCH updates/creates the configuration (upsert semantics)
|
||||
- No ID is required in the URL
|
||||
"""
|
||||
|
||||
queryset = LighthouseTenantConfiguration.objects.all()
|
||||
serializer_class = LighthouseTenantConfigSerializer
|
||||
http_method_names = ["get", "patch"]
|
||||
|
||||
def get_queryset(self):
|
||||
if getattr(self, "swagger_fake_view", False):
|
||||
return LighthouseTenantConfiguration.objects.none()
|
||||
return LighthouseTenantConfiguration.objects.filter(
|
||||
tenant_id=self.request.tenant_id
|
||||
)
|
||||
|
||||
def get_serializer_class(self):
|
||||
if self.action == "partial_update":
|
||||
return LighthouseTenantConfigUpdateSerializer
|
||||
return super().get_serializer_class()
|
||||
|
||||
def get_object(self):
|
||||
"""Retrieve the singleton instance for the current tenant."""
|
||||
obj = LighthouseTenantConfiguration.objects.filter(
|
||||
tenant_id=self.request.tenant_id
|
||||
).first()
|
||||
if obj is None:
|
||||
raise NotFound("Tenant Lighthouse configuration not found")
|
||||
self.check_object_permissions(self.request, obj)
|
||||
return obj
|
||||
|
||||
def list(self, request, *args, **kwargs):
|
||||
"""GET endpoint for singleton - returns single object, not an array."""
|
||||
instance = self.get_object()
|
||||
serializer = self.get_serializer(instance)
|
||||
return Response(serializer.data)
|
||||
|
||||
def partial_update(self, request, *args, **kwargs):
|
||||
"""PATCH endpoint for singleton - no pk required. Auto-creates if not exists."""
|
||||
# Auto-create tenant config if it doesn't exist (upsert semantics)
|
||||
instance, created = LighthouseTenantConfiguration.objects.get_or_create(
|
||||
tenant_id=self.request.tenant_id,
|
||||
defaults={},
|
||||
)
|
||||
|
||||
# Extract attributes from JSON:API payload
|
||||
try:
|
||||
payload = json.loads(request.body)
|
||||
attributes = payload.get("data", {}).get("attributes", {})
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
raise ValidationError("Invalid JSON:API payload")
|
||||
|
||||
serializer = self.get_serializer(instance, data=attributes, partial=True)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
serializer.save()
|
||||
read_serializer = LighthouseTenantConfigSerializer(
|
||||
instance, context=self.get_serializer_context()
|
||||
)
|
||||
return Response(read_serializer.data, status=status.HTTP_200_OK)
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="List all LLM models",
|
||||
description="List available LLM models per configured provider for the current tenant.",
|
||||
),
|
||||
retrieve=extend_schema(
|
||||
tags=["Lighthouse AI"],
|
||||
summary="Retrieve LLM model details",
|
||||
description="Get details for a specific LLM model.",
|
||||
),
|
||||
)
|
||||
class LighthouseProviderModelsViewSet(BaseRLSViewSet):
|
||||
queryset = LighthouseProviderModels.objects.all()
|
||||
serializer_class = LighthouseProviderModelsSerializer
|
||||
filterset_class = LighthouseProviderModelsFilter
|
||||
# Expose as read-only catalog collection
|
||||
http_method_names = ["get"]
|
||||
|
||||
def get_queryset(self):
|
||||
if getattr(self, "swagger_fake_view", False):
|
||||
return LighthouseProviderModels.objects.none()
|
||||
return LighthouseProviderModels.objects.filter(tenant_id=self.request.tenant_id)
|
||||
|
||||
def get_serializer_class(self):
|
||||
return super().get_serializer_class()
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
tags=["Processor"],
|
||||
|
||||
372
api/src/backend/tasks/jobs/lighthouse_providers.py
Normal file
372
api/src/backend/tasks/jobs/lighthouse_providers.py
Normal file
@@ -0,0 +1,372 @@
|
||||
from typing import Dict
|
||||
|
||||
import boto3
|
||||
import openai
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from celery.utils.log import get_task_logger
|
||||
|
||||
from api.models import LighthouseProviderConfiguration, LighthouseProviderModels
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
def _extract_openai_api_key(
|
||||
provider_cfg: LighthouseProviderConfiguration,
|
||||
) -> str | None:
|
||||
"""
|
||||
Safely extract the OpenAI API key from a provider configuration.
|
||||
|
||||
Args:
|
||||
provider_cfg (LighthouseProviderConfiguration): The provider configuration instance
|
||||
containing the credentials.
|
||||
|
||||
Returns:
|
||||
str | None: The API key string if present and valid, otherwise None.
|
||||
"""
|
||||
creds = provider_cfg.credentials_decoded
|
||||
if not isinstance(creds, dict):
|
||||
return None
|
||||
api_key = creds.get("api_key")
|
||||
if not isinstance(api_key, str) or not api_key:
|
||||
return None
|
||||
return api_key
|
||||
|
||||
|
||||
def _extract_bedrock_credentials(
|
||||
provider_cfg: LighthouseProviderConfiguration,
|
||||
) -> Dict[str, str] | None:
|
||||
"""
|
||||
Safely extract AWS Bedrock credentials from a provider configuration.
|
||||
|
||||
Args:
|
||||
provider_cfg (LighthouseProviderConfiguration): The provider configuration instance
|
||||
containing the credentials.
|
||||
|
||||
Returns:
|
||||
Dict[str, str] | None: Dictionary with 'access_key_id', 'secret_access_key', and
|
||||
'region' if present and valid, otherwise None.
|
||||
"""
|
||||
creds = provider_cfg.credentials_decoded
|
||||
if not isinstance(creds, dict):
|
||||
return None
|
||||
|
||||
access_key_id = creds.get("access_key_id")
|
||||
secret_access_key = creds.get("secret_access_key")
|
||||
region = creds.get("region")
|
||||
|
||||
# Validate all required fields are present and are strings
|
||||
if (
|
||||
not isinstance(access_key_id, str)
|
||||
or not access_key_id
|
||||
or not isinstance(secret_access_key, str)
|
||||
or not secret_access_key
|
||||
or not isinstance(region, str)
|
||||
or not region
|
||||
):
|
||||
return None
|
||||
|
||||
return {
|
||||
"access_key_id": access_key_id,
|
||||
"secret_access_key": secret_access_key,
|
||||
"region": region,
|
||||
}
|
||||
|
||||
|
||||
def check_lighthouse_provider_connection(provider_config_id: str) -> Dict:
|
||||
"""
|
||||
Validate a Lighthouse provider configuration by calling the provider API and
|
||||
toggle its active state accordingly.
|
||||
|
||||
Args:
|
||||
provider_config_id: The primary key of the `LighthouseProviderConfiguration`
|
||||
to validate.
|
||||
|
||||
Returns:
|
||||
dict: A result dictionary with the following keys:
|
||||
- "connected" (bool): Whether the provider credentials are valid.
|
||||
- "error" (str | None): The error message when not connected, otherwise None.
|
||||
|
||||
Side Effects:
|
||||
- Updates and persists `is_active` on the `LighthouseProviderConfiguration`.
|
||||
|
||||
Raises:
|
||||
LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID.
|
||||
"""
|
||||
provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id)
|
||||
|
||||
try:
|
||||
if (
|
||||
provider_cfg.provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
|
||||
):
|
||||
api_key = _extract_openai_api_key(provider_cfg)
|
||||
if not api_key:
|
||||
provider_cfg.is_active = False
|
||||
provider_cfg.save()
|
||||
return {"connected": False, "error": "API key is invalid or missing"}
|
||||
|
||||
# Test connection by listing models
|
||||
client = openai.OpenAI(api_key=api_key)
|
||||
_ = client.models.list()
|
||||
|
||||
elif (
|
||||
provider_cfg.provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
bedrock_creds = _extract_bedrock_credentials(provider_cfg)
|
||||
if not bedrock_creds:
|
||||
provider_cfg.is_active = False
|
||||
provider_cfg.save()
|
||||
return {
|
||||
"connected": False,
|
||||
"error": "AWS credentials are invalid or missing",
|
||||
}
|
||||
|
||||
# Test connection by listing foundation models
|
||||
bedrock_client = boto3.client(
|
||||
"bedrock",
|
||||
aws_access_key_id=bedrock_creds["access_key_id"],
|
||||
aws_secret_access_key=bedrock_creds["secret_access_key"],
|
||||
region_name=bedrock_creds["region"],
|
||||
)
|
||||
_ = bedrock_client.list_foundation_models()
|
||||
|
||||
else:
|
||||
return {"connected": False, "error": "Unsupported provider type"}
|
||||
|
||||
# Connection successful
|
||||
provider_cfg.is_active = True
|
||||
provider_cfg.save()
|
||||
return {"connected": True, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"%s connection check failed: %s", provider_cfg.provider_type, str(e)
|
||||
)
|
||||
provider_cfg.is_active = False
|
||||
provider_cfg.save()
|
||||
return {"connected": False, "error": str(e)}
|
||||
|
||||
|
||||
def _fetch_openai_models(api_key: str) -> Dict[str, str]:
|
||||
"""
|
||||
Fetch available models from OpenAI API.
|
||||
|
||||
Args:
|
||||
api_key: OpenAI API key for authentication.
|
||||
|
||||
Returns:
|
||||
Dict mapping model_id to model_name. For OpenAI, both are the same
|
||||
as the API doesn't provide separate display names.
|
||||
|
||||
Raises:
|
||||
Exception: If the API call fails.
|
||||
"""
|
||||
client = openai.OpenAI(api_key=api_key)
|
||||
models = client.models.list()
|
||||
# OpenAI uses model.id for both ID and display name
|
||||
return {m.id: m.id for m in getattr(models, "data", [])}
|
||||
|
||||
|
||||
def _fetch_bedrock_models(bedrock_creds: Dict[str, str]) -> Dict[str, str]:
|
||||
"""
|
||||
Fetch available models from AWS Bedrock with entitlement verification.
|
||||
|
||||
This function:
|
||||
1. Lists foundation models with TEXT modality support
|
||||
2. Lists inference profiles with TEXT modality support
|
||||
3. Verifies user has entitlement access to each model
|
||||
|
||||
Args:
|
||||
bedrock_creds: Dictionary with 'access_key_id', 'secret_access_key', and 'region'.
|
||||
|
||||
Returns:
|
||||
Dict mapping model_id to model_name for all accessible models.
|
||||
|
||||
Raises:
|
||||
BotoCoreError, ClientError: If AWS API calls fail.
|
||||
"""
|
||||
bedrock_client = boto3.client(
|
||||
"bedrock",
|
||||
aws_access_key_id=bedrock_creds["access_key_id"],
|
||||
aws_secret_access_key=bedrock_creds["secret_access_key"],
|
||||
region_name=bedrock_creds["region"],
|
||||
)
|
||||
|
||||
models_to_check: Dict[str, str] = {}
|
||||
|
||||
# Step 1: Get foundation models with TEXT modality
|
||||
foundation_response = bedrock_client.list_foundation_models()
|
||||
model_summaries = foundation_response.get("modelSummaries", [])
|
||||
|
||||
for model in model_summaries:
|
||||
# Check if model supports TEXT input and output modality
|
||||
input_modalities = model.get("inputModalities", [])
|
||||
output_modalities = model.get("outputModalities", [])
|
||||
|
||||
if "TEXT" not in input_modalities or "TEXT" not in output_modalities:
|
||||
continue
|
||||
|
||||
model_id = model.get("modelId")
|
||||
if not model_id:
|
||||
continue
|
||||
|
||||
inference_types = model.get("inferenceTypesSupported", [])
|
||||
|
||||
# Only include models with ON_DEMAND inference support
|
||||
if "ON_DEMAND" in inference_types:
|
||||
models_to_check[model_id] = model["modelName"]
|
||||
|
||||
# Step 2: Get inference profiles
|
||||
try:
|
||||
inference_profiles_response = bedrock_client.list_inference_profiles()
|
||||
inference_profiles = inference_profiles_response.get(
|
||||
"inferenceProfileSummaries", []
|
||||
)
|
||||
|
||||
for profile in inference_profiles:
|
||||
# Check if profile supports TEXT modality
|
||||
input_modalities = profile.get("inputModalities", [])
|
||||
output_modalities = profile.get("outputModalities", [])
|
||||
|
||||
if "TEXT" not in input_modalities or "TEXT" not in output_modalities:
|
||||
continue
|
||||
|
||||
profile_id = profile.get("inferenceProfileId")
|
||||
if profile_id:
|
||||
models_to_check[profile_id] = profile["inferenceProfileName"]
|
||||
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.info(
|
||||
"Could not fetch inference profiles in %s: %s",
|
||||
bedrock_creds["region"],
|
||||
str(e),
|
||||
)
|
||||
|
||||
# Step 3: Verify entitlement availability for each model
|
||||
available_models: Dict[str, str] = {}
|
||||
|
||||
for model_id, model_name in models_to_check.items():
|
||||
try:
|
||||
availability = bedrock_client.get_foundation_model_availability(
|
||||
modelId=model_id
|
||||
)
|
||||
|
||||
entitlement = availability.get("entitlementAvailability")
|
||||
|
||||
# Only include models user has access to
|
||||
if entitlement == "AVAILABLE":
|
||||
available_models[model_id] = model_name
|
||||
else:
|
||||
logger.debug(
|
||||
"Skipping model %s - entitlement status: %s", model_id, entitlement
|
||||
)
|
||||
|
||||
except (BotoCoreError, ClientError) as e:
|
||||
logger.debug(
|
||||
"Could not check availability for model %s: %s", model_id, str(e)
|
||||
)
|
||||
continue
|
||||
|
||||
return available_models
|
||||
|
||||
|
||||
def refresh_lighthouse_provider_models(provider_config_id: str) -> Dict:
|
||||
"""
|
||||
Refresh the catalog of models for a Lighthouse provider configuration.
|
||||
|
||||
Fetches the current list of models from the provider, upserts entries into
|
||||
`LighthouseProviderModels`, and deletes stale entries no longer returned.
|
||||
|
||||
Args:
|
||||
provider_config_id: The primary key of the `LighthouseProviderConfiguration`
|
||||
whose models should be refreshed.
|
||||
|
||||
Returns:
|
||||
dict: A result dictionary with the following keys on success:
|
||||
- "created" (int): Number of new model rows created.
|
||||
- "updated" (int): Number of existing model rows updated.
|
||||
- "deleted" (int): Number of stale model rows removed.
|
||||
If an error occurs, the dictionary will contain an "error" (str) field instead.
|
||||
|
||||
Raises:
|
||||
LighthouseProviderConfiguration.DoesNotExist: If no configuration exists with the given ID.
|
||||
"""
|
||||
provider_cfg = LighthouseProviderConfiguration.objects.get(pk=provider_config_id)
|
||||
fetched_models: Dict[str, str] = {}
|
||||
|
||||
# Fetch models from the appropriate provider
|
||||
try:
|
||||
if (
|
||||
provider_cfg.provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.OPENAI
|
||||
):
|
||||
api_key = _extract_openai_api_key(provider_cfg)
|
||||
if not api_key:
|
||||
return {
|
||||
"created": 0,
|
||||
"updated": 0,
|
||||
"deleted": 0,
|
||||
"error": "API key is invalid or missing",
|
||||
}
|
||||
fetched_models = _fetch_openai_models(api_key)
|
||||
|
||||
elif (
|
||||
provider_cfg.provider_type
|
||||
== LighthouseProviderConfiguration.LLMProviderChoices.BEDROCK
|
||||
):
|
||||
bedrock_creds = _extract_bedrock_credentials(provider_cfg)
|
||||
if not bedrock_creds:
|
||||
return {
|
||||
"created": 0,
|
||||
"updated": 0,
|
||||
"deleted": 0,
|
||||
"error": "AWS credentials are invalid or missing",
|
||||
}
|
||||
fetched_models = _fetch_bedrock_models(bedrock_creds)
|
||||
|
||||
else:
|
||||
return {
|
||||
"created": 0,
|
||||
"updated": 0,
|
||||
"deleted": 0,
|
||||
"error": "Unsupported provider type",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Unexpected error refreshing %s models: %s",
|
||||
provider_cfg.provider_type,
|
||||
str(e),
|
||||
)
|
||||
return {"created": 0, "updated": 0, "deleted": 0, "error": str(e)}
|
||||
|
||||
# Upsert models into the catalog
|
||||
created = 0
|
||||
updated = 0
|
||||
|
||||
for model_id, model_name in fetched_models.items():
|
||||
obj, was_created = LighthouseProviderModels.objects.update_or_create(
|
||||
tenant_id=provider_cfg.tenant_id,
|
||||
provider_configuration=provider_cfg,
|
||||
model_id=model_id,
|
||||
defaults={
|
||||
"model_name": model_name,
|
||||
"default_parameters": {},
|
||||
},
|
||||
)
|
||||
if was_created:
|
||||
created += 1
|
||||
else:
|
||||
updated += 1
|
||||
|
||||
# Delete stale models not present anymore
|
||||
deleted, _ = (
|
||||
LighthouseProviderModels.objects.filter(
|
||||
tenant_id=provider_cfg.tenant_id, provider_configuration=provider_cfg
|
||||
)
|
||||
.exclude(model_id__in=fetched_models.keys())
|
||||
.delete()
|
||||
)
|
||||
|
||||
return {"created": created, "updated": updated, "deleted": deleted}
|
||||
@@ -27,6 +27,10 @@ from tasks.jobs.integrations import (
|
||||
upload_s3_integration,
|
||||
upload_security_hub_integration,
|
||||
)
|
||||
from tasks.jobs.lighthouse_providers import (
|
||||
check_lighthouse_provider_connection,
|
||||
refresh_lighthouse_provider_models,
|
||||
)
|
||||
from tasks.jobs.report import generate_threatscore_report_job
|
||||
from tasks.jobs.scan import (
|
||||
aggregate_findings,
|
||||
@@ -524,6 +528,24 @@ def check_lighthouse_connection_task(lighthouse_config_id: str, tenant_id: str =
|
||||
return check_lighthouse_connection(lighthouse_config_id=lighthouse_config_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="lighthouse-provider-connection-check")
|
||||
@set_tenant
|
||||
def check_lighthouse_provider_connection_task(
|
||||
provider_config_id: str, tenant_id: str | None = None
|
||||
) -> dict:
|
||||
"""Task wrapper to validate provider credentials and set is_active."""
|
||||
return check_lighthouse_provider_connection(provider_config_id=provider_config_id)
|
||||
|
||||
|
||||
@shared_task(base=RLSTask, name="lighthouse-provider-models-refresh")
|
||||
@set_tenant
|
||||
def refresh_lighthouse_provider_models_task(
|
||||
provider_config_id: str, tenant_id: str | None = None
|
||||
) -> dict:
|
||||
"""Task wrapper to refresh provider models catalog for the given configuration."""
|
||||
return refresh_lighthouse_provider_models(provider_config_id=provider_config_id)
|
||||
|
||||
|
||||
@shared_task(name="integration-check")
|
||||
def check_integrations_task(tenant_id: str, provider_id: str, scan_id: str = None):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user