From 3bb3edc0bbc560f612db6ca2000e9efc1f0c9ad2 Mon Sep 17 00:00:00 2001 From: Chandrapal Badshah <12944530+Chan9390@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:36:25 +0530 Subject: [PATCH] feat: add caching recommendations in valkey --- ui/app/(prowler)/page.tsx | 20 ++ ui/components/lighthouse/banner.tsx | 31 ++- ui/lib/lighthouse/cache.ts | 369 +++++++++++++++++++++++++ ui/lib/lighthouse/recommendations.ts | 75 +++++ ui/lib/lighthouse/suggested-actions.ts | 33 +++ ui/lib/lighthouse/summary.ts | 23 +- ui/package.json | 1 + 7 files changed, 533 insertions(+), 19 deletions(-) create mode 100644 ui/lib/lighthouse/cache.ts create mode 100644 ui/lib/lighthouse/recommendations.ts create mode 100644 ui/lib/lighthouse/suggested-actions.ts diff --git a/ui/app/(prowler)/page.tsx b/ui/app/(prowler)/page.tsx index 9e949983b7..8a6d2463a9 100644 --- a/ui/app/(prowler)/page.tsx +++ b/ui/app/(prowler)/page.tsx @@ -23,8 +23,20 @@ import { SkeletonTableNewFindings } from "@/components/overview/new-findings-tab import { ContentLayout } from "@/components/ui"; import { DataTable } from "@/components/ui/table"; import { createDict } from "@/lib/helper"; +import { initializeTenantCache } from "@/lib/lighthouse/cache"; import { FindingProps, SearchParamsProps } from "@/types"; +const SSRCacheInitializer = async () => { + try { + // Initialize tenant cache, scan summary, and trigger recommendation generation + await initializeTenantCache(); + return null; + } catch (error) { + console.error("Error initializing cache:", error); + return null; + } +}; + export default function Home({ searchParams, }: { @@ -33,7 +45,15 @@ export default function Home({ const searchParamsKey = JSON.stringify(searchParams || {}); return ( +<<<<<<< HEAD +======= + + + + + +>>>>>>> e3c63c01b (feat: add caching recommendations in valkey)
diff --git a/ui/components/lighthouse/banner.tsx b/ui/components/lighthouse/banner.tsx index d8fdceadb4..a52e66d995 100644 --- a/ui/components/lighthouse/banner.tsx +++ b/ui/components/lighthouse/banner.tsx @@ -2,6 +2,7 @@ import { Bot } from "lucide-react"; import Link from "next/link"; import { getLighthouseConfig } from "@/actions/lighthouse/lighthouse"; +import { CacheService } from "@/lib/lighthouse/cache"; interface BannerConfig { message: string; @@ -37,14 +38,38 @@ export const LighthouseBanner = async () => { gradient: "bg-gradient-to-r from-green-500 to-blue-500 hover:from-green-600 hover:to-blue-600 focus:ring-green-500/50 dark:from-green-600 dark:to-blue-600 dark:hover:from-green-700 dark:hover:to-blue-700 dark:focus:ring-green-400/50", }); - } else { + } + + // Check if recommendation exists + const cachedRecommendations = await CacheService.getRecommendations(); + + if ( + cachedRecommendations.success && + cachedRecommendations.data && + cachedRecommendations.data.trim().length > 0 + ) { return renderBanner({ - message: "Use Lighthouse to review your findings and gain insights", + message: cachedRecommendations.data, href: "/lighthouse", gradient: - "bg-gradient-to-r from-green-500 to-blue-500 hover:from-green-600 hover:to-blue-600 focus:ring-green-500/50 dark:from-green-600 dark:to-blue-600 dark:hover:from-green-700 dark:hover:to-blue-700 dark:focus:ring-green-400/50", + "bg-gradient-to-r from-blue-500 to-purple-600 hover:from-blue-600 hover:to-purple-700 focus:ring-blue-500/50 dark:from-blue-600 dark:to-purple-700 dark:hover:from-blue-700 dark:hover:to-purple-800 dark:focus:ring-blue-400/50", }); } + + // Check if recommendation is being processed + const isProcessing = await CacheService.isRecommendationProcessing(); + + if (isProcessing) { + return renderBanner({ + message: "Lighthouse is reviewing your findings for insights", + href: "/lighthouse", + gradient: + "bg-gradient-to-r from-orange-500 to-yellow-500 hover:from-orange-600 hover:to-yellow-600 focus:ring-orange-500/50 dark:from-orange-600 dark:to-yellow-600 dark:hover:from-orange-700 dark:hover:to-yellow-700 dark:focus:ring-orange-400/50", + }); + } + + // Lighthouse configured but no recommendation and not processing - don't show banner + return null; } catch (error) { console.error("Error getting banner state:", error); return null; diff --git a/ui/lib/lighthouse/cache.ts b/ui/lib/lighthouse/cache.ts new file mode 100644 index 0000000000..5d66af11c6 --- /dev/null +++ b/ui/lib/lighthouse/cache.ts @@ -0,0 +1,369 @@ +import Valkey from "iovalkey"; + +import { auth } from "@/auth.config"; + +import { generateRecommendation } from "./recommendations"; +import { + generateSecurityScanSummary, + getCompletedScansLast24h, + compareProcessedScanIds, +} from "./summary"; + +let valkeyClient: Valkey | null = null; + +export async function getValkeyClient(): Promise { + if (!valkeyClient) { + valkeyClient = new Valkey({ + host: process.env.VALKEY_HOST, + port: parseInt(process.env.VALKEY_PORT || "6379"), + connectTimeout: 5000, + lazyConnect: true, + }); + } + + return valkeyClient; +} + +export class CacheService { + private static async getTenantId(): Promise { + const session = await auth(); + return session?.tenantId || null; + } + + private static async acquireProcessingLock( + tenantId: string, + lockKey: string, + lockTtlSeconds: number = 300, + ): Promise { + try { + const client = await getValkeyClient(); + const fullLockKey = `_lighthouse:${tenantId}:lock:${lockKey}`; + + const result = await client.set( + fullLockKey, + Date.now().toString(), + "EX", + lockTtlSeconds, + "NX", + ); + + return result === "OK"; + } catch (error) { + return false; + } + } + + private static async releaseProcessingLock( + tenantId: string, + lockKey: string, + ): Promise { + try { + const client = await getValkeyClient(); + const fullLockKey = `_lighthouse:${tenantId}:lock:${lockKey}`; + await client.del([fullLockKey]); + } catch (error) { + // Silent failure + } + } + + static async getProcessedScanIds(): Promise { + const tenantId = await this.getTenantId(); + if (!tenantId) return []; + + try { + const client = await getValkeyClient(); + const dataKey = `_lighthouse:${tenantId}:processed_scan_ids`; + + const result = await client.get(dataKey); + if (!result) return []; + + const scanIdsString = result.toString(); + return scanIdsString ? scanIdsString.split(",") : []; + } catch (error) { + return []; + } + } + + static async setProcessedScanIds(scanIds: string[]): Promise { + const tenantId = await this.getTenantId(); + if (!tenantId) return false; + + try { + const client = await getValkeyClient(); + const dataKey = `_lighthouse:${tenantId}:processed_scan_ids`; + const scanIdsString = scanIds.join(","); + + await client.set(dataKey, scanIdsString); + return true; + } catch (error) { + return false; + } + } + + static async processScansWithLock(scanIds: string[]): Promise<{ + success: boolean; + data?: string; + }> { + const tenantId = await this.getTenantId(); + if (!tenantId) return { success: false }; + + const lockKey = "scan-processing"; + const lockTtlSeconds = 1200; // 20 minutes + + try { + // Try to acquire processing lock + const lockAcquired = await this.acquireProcessingLock( + tenantId, + lockKey, + lockTtlSeconds, + ); + + if (!lockAcquired) { + // Processing is happening in background, return success but no data + return { success: true }; + } + + try { + // Generate the scan summary for the provided scan IDs + const scanSummary = await generateSecurityScanSummary(scanIds); + + // Only process if we have valid scan summary + if (scanSummary) { + // Cache the scan summary + await this.set("scan-summary", scanSummary); + + // Mark scans as processed + await this.setProcessedScanIds(scanIds); + + // Generate and cache recommendations asynchronously + this.generateAndCacheRecommendations(scanSummary) + .then((result) => { + if (result.success && result.data) { + console.log("Background recommendation generated successfully"); + } + }) + .catch((error) => { + console.error( + "Background recommendation generation failed:", + error, + ); + }); + + return { + success: true, + data: scanSummary, + }; + } else { + // Even if no summary, mark scans as processed to avoid reprocessing + await this.setProcessedScanIds(scanIds); + } + + return { success: true }; + } finally { + await this.releaseProcessingLock(tenantId, lockKey); + } + } catch (error) { + console.error("Error processing scans with lock:", error); + return { success: false }; + } + } + + // Generic cache methods for future use + static async get(key: string): Promise { + const tenantId = await this.getTenantId(); + if (!tenantId) return null; + + try { + const client = await getValkeyClient(); + const fullKey = `_lighthouse:${tenantId}:${key}`; + const result = await client.get(fullKey); + return result?.toString() || null; + } catch (error) { + return null; + } + } + + static async set( + key: string, + value: string, + ttlSeconds?: number, + ): Promise { + const tenantId = await this.getTenantId(); + if (!tenantId) return false; + + try { + const client = await getValkeyClient(); + const fullKey = `_lighthouse:${tenantId}:${key}`; + + if (ttlSeconds) { + await client.set(fullKey, value, "EX", ttlSeconds); + } else { + await client.set(fullKey, value); + } + + return true; + } catch (error) { + return false; + } + } + + static async getCachedMessage(): Promise { + return await this.get("scan-summary"); + } + + static async getRecommendations(): Promise<{ + success: boolean; + data?: string; + }> { + const tenantId = await this.getTenantId(); + if (!tenantId) return { success: false }; + + try { + const client = await getValkeyClient(); + const dataKey = `_lighthouse:${tenantId}:recommendations`; + + const cachedData = await client.get(dataKey); + if (cachedData) { + return { + success: true, + data: cachedData.toString(), + }; + } + + return { success: true, data: undefined }; + } catch (error) { + return { success: false }; + } + } + + static async generateAndCacheRecommendations(scanSummary: string): Promise<{ + success: boolean; + data?: string; + }> { + const tenantId = await this.getTenantId(); + if (!tenantId) return { success: false }; + + const lockKey = "recommendations-processing"; + const dataKey = `_lighthouse:${tenantId}:recommendations`; + + try { + const client = await getValkeyClient(); + + // Check if data already exists + const existingData = await client.get(dataKey); + if (existingData) { + return { + success: true, + data: existingData.toString(), + }; + } + + // Lock TTL 10 minutes + const lockAcquired = await this.acquireProcessingLock( + tenantId, + lockKey, + 600, + ); + + if (!lockAcquired) { + // Processing is happening in background, return success but no data + return { success: true }; + } + + try { + // Double-check after acquiring lock + const doubleCheckData = await client.get(dataKey); + if (doubleCheckData) { + return { + success: true, + data: doubleCheckData.toString(), + }; + } + + // Generate recommendation using LLM + const recommendation = await generateRecommendation(scanSummary); + + // Only cache non-empty recommendations + if (recommendation.trim()) { + await client.set(dataKey, recommendation); + } + + return { + success: true, + data: recommendation, + }; + } finally { + await this.releaseProcessingLock(tenantId, lockKey); + } + } catch (error) { + console.error("Error generating and caching recommendations:", error); + return { success: false }; + } + } + + static async isRecommendationProcessing(): Promise { + const tenantId = await this.getTenantId(); + if (!tenantId) return false; + + try { + const client = await getValkeyClient(); + const lockKey = `_lighthouse:${tenantId}:lock:recommendations-processing`; + + const result = await client.get(lockKey); + return result !== null; + } catch (error) { + return false; + } + } +} + +export async function initializeTenantCache(): Promise<{ + success: boolean; + data?: string; + scanSummary?: string; +}> { + try { + // Quick pre-check: Do we need to process anything? + const currentScanIds = await getCompletedScansLast24h(); + + if (currentScanIds.length === 0) { + // No scans in last 24h, return existing cached data if any + const existingSummary = await CacheService.get("scan-summary"); + return { + success: true, + data: existingSummary || undefined, + scanSummary: existingSummary || undefined, + }; + } + + // Check if we need to process these scans + const processedScanIds = await CacheService.getProcessedScanIds(); + const shouldProcess = !compareProcessedScanIds( + currentScanIds, + processedScanIds, + ); + + if (!shouldProcess) { + // Scans already processed, return existing cached data + const existingSummary = await CacheService.get("scan-summary"); + return { + success: true, + data: existingSummary || undefined, + scanSummary: existingSummary || undefined, + }; + } + + // New scans found, trigger full processing with lock + const result = await CacheService.processScansWithLock(currentScanIds); + return { + success: result.success, + data: result.data, + scanSummary: result.data, + }; + } catch (error) { + console.error("Error initializing tenant cache:", error); + return { + success: false, + }; + } +} diff --git a/ui/lib/lighthouse/recommendations.ts b/ui/lib/lighthouse/recommendations.ts new file mode 100644 index 0000000000..bf2837699b --- /dev/null +++ b/ui/lib/lighthouse/recommendations.ts @@ -0,0 +1,75 @@ +import { ChatOpenAI } from "@langchain/openai"; + +import { getAIKey, getLighthouseConfig } from "@/actions/lighthouse/lighthouse"; + +export const generateRecommendation = async ( + scanSummary: string, +): Promise => { + try { + const apiKey = await getAIKey(); + if (!apiKey) { + return ""; + } + + // Get lighthouse configuration + const lighthouseConfig = await getLighthouseConfig(); + if (!lighthouseConfig?.attributes) { + return ""; + } + + const config = lighthouseConfig.attributes; + const finalBusinessContext = config.business_context || ""; + + const llm = new ChatOpenAI({ + model: config.model || "gpt-4o", + temperature: config.temperature || 0, + maxTokens: 150, + apiKey: apiKey, + }); + + // Build the prompt with business context awareness + let systemPrompt = `You are a cloud security analyst creating concise business recommendations for a banner notification. + +IMPORTANT: Your response must be a single, short sentence (max 80 characters) that would make a user want to click on a banner to learn more. + +GUIDELINES: +- Frame recommendations in business terms, not technical jargon +- Focus on actionable insights +- Make it clickable and engaging +- Don't use phrases like "Lighthouse says" or "Lighthouse recommends" +- Be specific about the type of improvement when possible +- Use only information from the security scan summary to generate the recommendation +- Add words like "Lighthouse" to the recommendation +- Don't end with a question mark or full stop +- Don't use words like "urges" or "requires" +- Don't wrap the message in double quotes or single quotes +- Use words like "detected" or "found" to describe the issue + +EXAMPLES OF GOOD RESPONSES: +- Lighthouse detected critical issues in authentication services +- Lighthouse found a new exposed S3 bucket in recent scan +- Lighthouse identified fixing one check could resolve 30 open findings + +Based on the below security scan summary, generate ONE short business recommendation:`; + + if (finalBusinessContext) { + systemPrompt += `\n\nBUSINESS CONTEXT: ${finalBusinessContext}`; + } + + systemPrompt += `\n\nBased on this security scan summary, generate 1 engaging banner message:\n\n${scanSummary}`; + + const response = await llm.invoke([ + { + role: "system", + content: systemPrompt, + }, + ]); + + const recommendation = response.content.toString().trim(); + + return recommendation.length > 0 ? recommendation : ""; + } catch (error) { + console.error("Error generating recommendation:", error); + return ""; + } +}; diff --git a/ui/lib/lighthouse/suggested-actions.ts b/ui/lib/lighthouse/suggested-actions.ts new file mode 100644 index 0000000000..79048b92d6 --- /dev/null +++ b/ui/lib/lighthouse/suggested-actions.ts @@ -0,0 +1,33 @@ +export interface SuggestedAction { + title: string; + label: string; + action: string; + questionRef?: string; +} + +export const suggestedActions: SuggestedAction[] = [ + { + title: "Are there any exposed S3", + label: "buckets in my AWS accounts?", + action: "List exposed S3 buckets in my AWS accounts", + questionRef: "1", + }, + { + title: "What is the risk of having", + label: "RDS databases unencrypted?", + action: "What is the risk of having RDS databases unencrypted?", + questionRef: "2", + }, + { + title: "What is the CIS 1.10 compliance status", + label: "of my Kubernetes cluster?", + action: "What is the CIS 1.10 compliance status of my Kubernetes cluster?", + questionRef: "3", + }, + { + title: "List my highest privileged", + label: "AWS IAM users with full admin access?", + action: "List my highest privileged AWS IAM users with full admin access", + questionRef: "4", + }, +]; diff --git a/ui/lib/lighthouse/summary.ts b/ui/lib/lighthouse/summary.ts index 6ea55c4073..e6dfe29154 100644 --- a/ui/lib/lighthouse/summary.ts +++ b/ui/lib/lighthouse/summary.ts @@ -5,7 +5,7 @@ import { CheckDetails, FindingSummary } from "@/types/lighthouse/summary"; import { getNewFailedFindingsSummary } from "./tools/findings"; -const getCompletedScansLast24h = async (): Promise => { +export const getCompletedScansLast24h = async (): Promise => { const twentyFourHoursAgo = new Date(); twentyFourHoursAgo.setHours(twentyFourHoursAgo.getHours() - 24); @@ -27,7 +27,7 @@ const getCompletedScansLast24h = async (): Promise => { return scansResponse.data.map((scan: any) => scan.id); }; -const compareProcessedScanIds = ( +export const compareProcessedScanIds = ( currentScanIds: string[], processedScanIds: string[], ): boolean => { @@ -229,22 +229,13 @@ const buildSingleFindingDetails = ( return detailsText; }; -// Generates a summary of failed findings from security scans in last 24 hours -// Returns an empty string if - no scans in 24 hours, no failed findings in any scan, or unexpected error +// Generates a summary of failed findings for the provided scan IDs +// Returns an empty string if no failed findings in any scan or unexpected error // Else it returns a string with the summary of the failed findings -export const generateSecurityScanSummary = async (): Promise => { +export const generateSecurityScanSummary = async ( + scanIds: string[], +): Promise => { try { - const currentScanIds = await getCompletedScansLast24h(); - - if (currentScanIds.length === 0) { - return ""; - } - - // TODO: Check if these scan IDs were already processed - // This will be implemented in later steps when we update the cache service - - const scanIds = currentScanIds; - // Collect new failed findings by scan const newFindingsByScan = await collectNewFailedFindings(scanIds); diff --git a/ui/package.json b/ui/package.json index da794f3b33..34384a07c1 100644 --- a/ui/package.json +++ b/ui/package.json @@ -31,6 +31,7 @@ "framer-motion": "^11.16.0", "immer": "^10.1.1", "intl-messageformat": "^10.5.0", + "iovalkey": "^0.3.3", "jose": "^5.9.3", "js-yaml": "^4.1.0", "jwt-decode": "^4.0.0",