ben.carrasco/prowler
1
0
Fork 0
mirror of https://github.com/prowler-cloud/prowler.git synced 2026-10-05 03:12:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): add Lighthouse request outcome feedback (#12419)

Browse Source 
Co-authored-by: alejandrobailo <alejandrobailo94@gmail.com>
This commit is contained in:
Alan Buscaglia
2026-08-26 11:34:40 +02:00
committed by GitHub
co-authored by alejandrobailo
parent 6449f3a592
commit 95642fb220
15 changed files with 2041 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/app/(prowler)/lighthouse/_actions/lighthouse-v2.adapter.test.ts
+21
View File
@@ -122,6 +122,27 @@ describe("lighthouse-v2.adapter", () => {
});
});
it("should not expose removed feedback state on mapped messages", () => {
// Given
const resource: Parameters<typeof mapLighthouseV2Message>[0] = {
id: "message-1",
type: "lighthouse-messages",
attributes: {
role: "user",
model: null,
token_usage: null,
inserted_at: "2026-06-24T10:01:00Z",
parts: [],
},
};
// When
const message = mapLighthouseV2Message(resource);
// Then
expect(message).not.toHaveProperty("feedback");
});
it("should give id-less parts stable fallback keys instead of empty strings", () => {
// Given
const resource: Parameters<typeof mapLighthouseV2Message>[0] = {
ui/app/(prowler)/lighthouse/_actions/lighthouse-v2.ts
+2 -1
View File
@@ -345,11 +345,12 @@ async function mutateEmpty(
path: string,
init: RequestInit,
pathToRevalidate: string,
includeContentType = false,
): Promise<LighthouseV2ActionResult<true>> {
try {
const response = await fetch(buildApiUrl(path), {
...init,
headers: await getAuthHeaders({ contentType: false }),
headers: await getAuthHeaders({ contentType: includeContentType }),
});
const document = await handleApiResponse(response, pathToRevalidate);
if (isErrorDocument(document)) {
ui/app/(prowler)/lighthouse/_components/chat/composer.tsx
+27 -5
View File
@@ -4,6 +4,7 @@ import { CornerDownLeft, Settings, TriangleAlert } from "lucide-react";
import Link from "next/link";
import { type ReactNode, type SubmitEvent, useRef } from "react";
import type { LighthouseV2Message } from "@/app/(prowler)/lighthouse/_types";
import { Alert, AlertDescription } from "@/components/shadcn/alert";
import { Button } from "@/components/shadcn/button/button";
import { Spinner } from "@/components/shadcn/spinner/spinner";
@@ -16,8 +17,13 @@ import {
import { useMountEffect } from "@/hooks/use-mount-effect";
import { LIGHTHOUSE_ROUTE } from "@/lib/lighthouse-routes";
import type { LighthouseFeedbackSurvey } from "./lighthouse-feedback-survey";
import { LighthouseOutcomeFeedbackControls } from "./message-bubble";
interface ChatComposerPanelProps {
feedback: string | null;
feedbackTarget?: LighthouseV2Message;
feedbackSurvey?: LighthouseFeedbackSurvey | null;
canRetry: boolean;
onRetry: () => void;
onDismissFeedback: () => void;
@@ -36,6 +42,8 @@ interface ChatComposerPanelProps {
// two branches can't drift apart.
export function ChatComposerPanel({
feedback,
feedbackTarget,
feedbackSurvey,
canRetry,
onRetry,
onDismissFeedback,
@@ -45,6 +53,8 @@ export function ChatComposerPanel({
<>
<ChatFeedbackBar
feedback={feedback}
feedbackTarget={feedbackTarget}
feedbackSurvey={feedbackSurvey}
canRetry={canRetry}
onRetry={onRetry}
onDismiss={onDismissFeedback}
@@ -56,11 +66,15 @@ export function ChatComposerPanel({
function ChatFeedbackBar({
feedback,
feedbackTarget,
feedbackSurvey,
canRetry,
onRetry,
onDismiss,
}: {
feedback: string | null;
feedbackTarget?: LighthouseV2Message;
feedbackSurvey?: LighthouseFeedbackSurvey | null;
canRetry: boolean;
onRetry: () => void;
onDismiss: () => void;
@@ -72,11 +86,19 @@ function ChatFeedbackBar({
<TriangleAlert />
<AlertDescription className="flex items-center justify-between gap-3">
<span>{feedback}</span>
{canRetry && (
<Button type="button" variant="outline" size="sm" onClick={onRetry}>
Retry
</Button>
)}
<div className="flex items-center gap-2">
{feedbackTarget && feedbackSurvey && (
<LighthouseOutcomeFeedbackControls
message={feedbackTarget}
survey={feedbackSurvey}
/>
)}
{canRetry && (
<Button type="button" variant="outline" size="sm" onClick={onRetry}>
Retry
</Button>
)}
</div>
</AlertDescription>
</Alert>
);
ui/app/(prowler)/lighthouse/_components/chat/lighthouse-feedback-survey.test.ts
+509
View File
@@ -0,0 +1,509 @@
import type { Survey } from "posthog-js";
import { describe, expect, it } from "vitest";
import {
buildLighthouseFeedbackSurveyEvents,
LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH,
type LighthouseFeedbackReason,
LIGHTHOUSE_FEEDBACK_REASONS,
resolveLighthouseFeedbackSurvey,
} from "./lighthouse-feedback-survey";
const SURVEY = {
id: "survey-123",
name: "Lighthouse Request Outcome Feedback",
type: "api",
questions: [
{
id: "rating-question-id",
type: "rating",
display: "emoji",
scale: 2,
question: "How was this outcome?",
lowerBoundLabel: "Not helpful",
upperBoundLabel: "Helpful",
},
{
id: "reasons-question-id",
type: "multiple_choice",
optional: true,
choices: [
"Don't like the style",
"Didn't fully follow instructions",
"Low quality",
"Biased",
"Safety or legal concern",
"Other",
],
question: "What could be improved?",
},
{
id: "details-question-id",
type: "open",
optional: true,
question: "Additional feedback",
},
],
} as unknown as Survey;
describe("Lighthouse outcome feedback survey", () => {
it("should build one completed rating snapshot for a helpful outcome", () => {
// Given
const survey = resolveLighthouseFeedbackSurvey([SURVEY]);
expect(survey).not.toBeNull();
// When
const events = buildLighthouseFeedbackSurveyEvents(survey!, {
targetMessageId: "message-user-1",
rating: "up",
submissionId: "submission-1",
});
// Then
expect(events).toEqual([
{
$survey_id: "survey-123",
$survey_name: "Lighthouse Request Outcome Feedback",
$survey_submission_id: "submission-1",
$survey_completed: true,
$ai_trace_id: "message-user-1",
$survey_questions: [
{
id: "rating-question-id",
question: "How was this outcome?",
response: 1,
},
],
"$survey_response_rating-question-id": 1,
$survey_response: 1,
},
]);
});
it("should emit cumulative snapshots for a negative outcome with optional answers", () => {
// Given
const survey = resolveLighthouseFeedbackSurvey([SURVEY]);
expect(survey).not.toBeNull();
// When
const events = buildLighthouseFeedbackSurveyEvents(survey!, {
targetMessageId: "message-user-1",
rating: "down",
reasons: ["Low quality", "Other"],
details: " Missing evidence. ",
submissionId: "submission-1",
});
// Then
expect(events).toEqual([
{
$survey_id: "survey-123",
$survey_name: "Lighthouse Request Outcome Feedback",
$survey_submission_id: "submission-1",
$survey_completed: false,
$ai_trace_id: "message-user-1",
$survey_questions: [
{
id: "rating-question-id",
question: "How was this outcome?",
response: 2,
},
],
"$survey_response_rating-question-id": 2,
$survey_response: 2,
},
{
$survey_id: "survey-123",
$survey_name: "Lighthouse Request Outcome Feedback",
$survey_submission_id: "submission-1",
$survey_completed: false,
$ai_trace_id: "message-user-1",
$survey_questions: [
{
id: "rating-question-id",
question: "How was this outcome?",
response: 2,
},
{
id: "reasons-question-id",
question: "What could be improved?",
response: ["Low quality", "Other"],
},
],
"$survey_response_rating-question-id": 2,
"$survey_response_reasons-question-id": ["Low quality", "Other"],
$survey_response: ["Low quality", "Other"],
},
{
$survey_id: "survey-123",
$survey_name: "Lighthouse Request Outcome Feedback",
$survey_submission_id: "submission-1",
$survey_completed: true,
$ai_trace_id: "message-user-1",
$survey_questions: [
{
id: "rating-question-id",
question: "How was this outcome?",
response: 2,
},
{
id: "reasons-question-id",
question: "What could be improved?",
response: ["Low quality", "Other"],
},
{
id: "details-question-id",
question: "Additional feedback",
response: "Missing evidence.",
},
],
"$survey_response_rating-question-id": 2,
"$survey_response_reasons-question-id": ["Low quality", "Other"],
"$survey_response_details-question-id": "Missing evidence.",
$survey_response: "Missing evidence.",
},
]);
});
it("should reject an API survey whose reason choices are not allowlisted", () => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
SURVEY.questions[0],
{
...SURVEY.questions[1],
choices: ["Not an allowlisted reason"],
},
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it("should reject an API survey whose reasons question has no choices", () => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
SURVEY.questions[0],
{ ...SURVEY.questions[1], choices: undefined },
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it("should reject an API survey whose reason choices are not an array", () => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
SURVEY.questions[0],
{ ...SURVEY.questions[1], choices: "Low quality" },
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it("should reject an API survey with an optional rating question", () => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
{ ...SURVEY.questions[0], optional: true },
SURVEY.questions[1],
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it("should reject an API survey with duplicate question identifiers", () => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
SURVEY.questions[0],
{ ...SURVEY.questions[1], id: SURVEY.questions[0].id },
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it.each([
["a non-rating type", { ...SURVEY.questions[0], type: "open" }],
["a non-emoji display", { ...SURVEY.questions[0], display: "number" }],
["a scale other than two", { ...SURVEY.questions[0], scale: 3 }],
])("should reject an API survey with %s", (_description, ratingQuestion) => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [ratingQuestion, SURVEY.questions[1], SURVEY.questions[2]],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it("should accept a required rating question that explicitly declares optional false", () => {
// Given
const surveyWithRequiredRating = {
...SURVEY,
questions: [
{ ...SURVEY.questions[0], optional: false },
SURVEY.questions[1],
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([surveyWithRequiredRating]);
// Then
expect(survey).not.toBeNull();
});
it.each([
["reasons", 1],
["details", 2],
])(
"should reject an API survey with a non-optional %s question",
(_name, index) => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
SURVEY.questions[0],
index === 1
? { ...SURVEY.questions[1], optional: false }
: SURVEY.questions[1],
index === 2
? { ...SURVEY.questions[2], optional: false }
: SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
},
);
it.each([
["missing", undefined],
["empty", ""],
["blank", " "],
])(
"should reject an API survey with a %s question identifier",
(_name, id) => {
// Given
const invalidSurvey = surveyWithQuestionIds([
SURVEY.questions[0].id,
id,
SURVEY.questions[2].id,
]);
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
},
);
it.each([
[
"rating and reasons",
"duplicate-rating-question-id",
"duplicate-rating-question-id",
SURVEY.questions[2].id,
],
[
"rating and details",
"duplicate-rating-question-id",
SURVEY.questions[1].id,
"duplicate-rating-question-id",
],
[
"reasons and details",
SURVEY.questions[0].id,
"duplicate-reasons-question-id",
"duplicate-reasons-question-id",
],
])(
"should reject an API survey with duplicate %s identifiers",
(_name, ratingId, reasonsId, detailsId) => {
// Given
const invalidSurvey = surveyWithQuestionIds([
ratingId,
reasonsId,
detailsId,
]);
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
},
);
it("should reject an API survey with reordered reason choices", () => {
// Given
const invalidSurvey = {
...SURVEY,
questions: [
SURVEY.questions[0],
{
...SURVEY.questions[1],
choices: [...LIGHTHOUSE_FEEDBACK_REASONS].reverse(),
},
SURVEY.questions[2],
],
} as unknown as Survey;
// When
const survey = resolveLighthouseFeedbackSurvey([invalidSurvey]);
// Then
expect(survey).toBeNull();
});
it("should deduplicate and allowlist reason responses in their selected order", () => {
// Given
const survey = resolveLighthouseFeedbackSurvey([SURVEY]);
expect(survey).not.toBeNull();
// When
const events = buildLighthouseFeedbackSurveyEvents(survey!, {
targetMessageId: "message-user-1",
rating: "down",
reasons: [
"Other",
"Low quality",
"Other",
"Unrecognized",
] as unknown as LighthouseFeedbackReason[],
submissionId: "submission-1",
});
// Then
expect(events).toHaveLength(2);
expect(events.at(-1)).toMatchObject({
$survey_completed: true,
"$survey_response_reasons-question-id": ["Other", "Low quality"],
});
});
it("should trim and defensively bound detail responses", () => {
// Given
const survey = resolveLighthouseFeedbackSurvey([SURVEY]);
expect(survey).not.toBeNull();
const details = ` ${"x".repeat(LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH + 1)} `;
// When
const events = buildLighthouseFeedbackSurveyEvents(survey!, {
targetMessageId: "message-user-1",
rating: "down",
details,
submissionId: "submission-1",
});
// Then
expect(events).toHaveLength(2);
expect(events.at(-1)).toMatchObject({
$survey_completed: true,
"$survey_response_details-question-id": "x".repeat(
LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH,
),
});
});
it("should complete a negative submission with reasons only", () => {
// Given
const survey = resolveLighthouseFeedbackSurvey([SURVEY]);
expect(survey).not.toBeNull();
// When
const events = buildLighthouseFeedbackSurveyEvents(survey!, {
targetMessageId: "message-user-1",
rating: "down",
reasons: ["Low quality"],
submissionId: "submission-1",
});
// Then
expect(events.map((event) => event.$survey_completed)).toEqual([
false,
true,
]);
expect(events.at(-1)?.$survey_questions).toHaveLength(2);
expect(events.at(-1)?.$survey_response).toEqual(["Low quality"]);
});
it("should complete a negative submission with details only", () => {
// Given
const survey = resolveLighthouseFeedbackSurvey([SURVEY]);
expect(survey).not.toBeNull();
// When
const events = buildLighthouseFeedbackSurveyEvents(survey!, {
targetMessageId: "message-user-1",
rating: "down",
details: "Missing evidence",
submissionId: "submission-1",
});
// Then
expect(events.map((event) => event.$survey_completed)).toEqual([
false,
true,
]);
expect(events.at(-1)?.$survey_questions).toHaveLength(2);
expect(events.at(-1)?.$survey_response).toBe("Missing evidence");
});
});
function surveyWithQuestionIds(ids: [unknown, unknown, unknown]): Survey {
return {
...SURVEY,
questions: [
{ ...SURVEY.questions[0], id: ids[0] },
{ ...SURVEY.questions[1], id: ids[1] },
{ ...SURVEY.questions[2], id: ids[2] },
],
} as unknown as Survey;
}
ui/app/(prowler)/lighthouse/_components/chat/lighthouse-feedback-survey.ts
+265
View File
@@ -0,0 +1,265 @@
import type { Survey, SurveyQuestion } from "posthog-js";
export const LIGHTHOUSE_FEEDBACK_SURVEY_NAME =
"Lighthouse Request Outcome Feedback";
export const LIGHTHOUSE_FEEDBACK_RATING = {
UP: "up",
DOWN: "down",
} as const;
export type LighthouseFeedbackRating =
(typeof LIGHTHOUSE_FEEDBACK_RATING)[keyof typeof LIGHTHOUSE_FEEDBACK_RATING];
export const LIGHTHOUSE_FEEDBACK_REASON = {
STYLE: "Don't like the style",
INSTRUCTIONS: "Didn't fully follow instructions",
QUALITY: "Low quality",
BIAS: "Biased",
SAFETY_OR_LEGAL: "Safety or legal concern",
OTHER: "Other",
} as const;
export type LighthouseFeedbackReason =
(typeof LIGHTHOUSE_FEEDBACK_REASON)[keyof typeof LIGHTHOUSE_FEEDBACK_REASON];
export const LIGHTHOUSE_FEEDBACK_REASONS = Object.values(
LIGHTHOUSE_FEEDBACK_REASON,
);
export const LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH = 2000;
interface LighthouseFeedbackQuestion {
id: string;
question: string;
}
export interface LighthouseFeedbackSurvey {
id: string;
name: string;
ratingQuestion: LighthouseFeedbackQuestion;
reasonsQuestion: LighthouseFeedbackQuestion;
detailsQuestion: LighthouseFeedbackQuestion;
}
interface LighthouseFeedbackSubmission {
targetMessageId: string;
rating: LighthouseFeedbackRating;
submissionId: string;
reasons?: LighthouseFeedbackReason[];
details?: string;
}
type LighthouseFeedbackResponse = number | string | LighthouseFeedbackReason[];
interface LighthouseFeedbackQuestionResponse {
id: string;
question: string;
response: LighthouseFeedbackResponse;
}
export interface LighthouseFeedbackSurveyEvent {
$survey_id: string;
$survey_name: string;
$survey_submission_id: string;
$survey_completed: boolean;
$ai_trace_id: string;
$survey_questions: LighthouseFeedbackQuestionResponse[];
$survey_response: LighthouseFeedbackResponse;
[responseProperty: `$survey_response_${string}`]: LighthouseFeedbackResponse;
}
export function resolveLighthouseFeedbackSurvey(
surveys: Survey[],
): LighthouseFeedbackSurvey | null {
const survey = surveys.find(
(item) =>
item.name === LIGHTHOUSE_FEEDBACK_SURVEY_NAME && item.type === "api",
);
if (
!survey ||
!Array.isArray(survey.questions) ||
survey.questions.length !== 3
) {
return null;
}
const [ratingQuestion, reasonsQuestion, detailsQuestion] = survey.questions;
if (
!isRatingQuestion(ratingQuestion) ||
!isReasonsQuestion(reasonsQuestion) ||
!isDetailsQuestion(detailsQuestion)
) {
return null;
}
if (
new Set([ratingQuestion.id, reasonsQuestion.id, detailsQuestion.id])
.size !== 3
) {
return null;
}
return {
id: survey.id,
name: survey.name,
ratingQuestion: toFeedbackQuestion(ratingQuestion),
reasonsQuestion: toFeedbackQuestion(reasonsQuestion),
detailsQuestion: toFeedbackQuestion(detailsQuestion),
};
}
export function buildLighthouseFeedbackSurveyEvents(
survey: LighthouseFeedbackSurvey,
submission: LighthouseFeedbackSubmission,
): LighthouseFeedbackSurveyEvent[] {
const ratingResponse =
submission.rating === LIGHTHOUSE_FEEDBACK_RATING.UP ? 1 : 2;
const reasons = normalizeReasons(submission.reasons);
const details = normalizeDetails(submission.details);
const ratingQuestion = toQuestionResponse(
survey.ratingQuestion,
ratingResponse,
);
const isNegative = submission.rating === LIGHTHOUSE_FEEDBACK_RATING.DOWN;
const events = [
buildSurveyEvent(
survey,
submission,
[ratingQuestion],
!isNegative || (reasons.length === 0 && !details),
),
];
if (isNegative && reasons.length > 0) {
events.push(
buildSurveyEvent(
survey,
submission,
[ratingQuestion, toQuestionResponse(survey.reasonsQuestion, reasons)],
!details,
),
);
}
if (isNegative && details) {
events.push(
buildSurveyEvent(
survey,
submission,
[
ratingQuestion,
...(reasons.length > 0
? [toQuestionResponse(survey.reasonsQuestion, reasons)]
: []),
toQuestionResponse(survey.detailsQuestion, details),
],
true,
),
);
}
return events;
}
function buildSurveyEvent(
survey: LighthouseFeedbackSurvey,
submission: LighthouseFeedbackSubmission,
questions: LighthouseFeedbackQuestionResponse[],
completed: boolean,
): LighthouseFeedbackSurveyEvent {
const latestResponse = questions.at(-1)!;
const responseProperties: Record<
`$survey_response_${string}`,
LighthouseFeedbackResponse
> = {};
for (const question of questions) {
responseProperties[`$survey_response_${question.id}`] = question.response;
}
return {
$survey_id: survey.id,
$survey_name: survey.name,
$survey_submission_id: submission.submissionId,
$survey_completed: completed,
$ai_trace_id: submission.targetMessageId,
$survey_questions: questions,
...responseProperties,
$survey_response: latestResponse.response,
};
}
function toQuestionResponse(
question: LighthouseFeedbackQuestion,
response: LighthouseFeedbackResponse,
): LighthouseFeedbackQuestionResponse {
return { ...question, response };
}
function normalizeReasons(
reasons: LighthouseFeedbackReason[] | undefined,
): LighthouseFeedbackReason[] {
const selected = new Set<LighthouseFeedbackReason>();
for (const reason of reasons ?? []) {
if (LIGHTHOUSE_FEEDBACK_REASONS.includes(reason)) selected.add(reason);
}
return Array.from(selected);
}
function normalizeDetails(details: string | undefined): string | undefined {
const trimmed = details
?.trim()
.slice(0, LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH);
return trimmed || undefined;
}
function isRatingQuestion(
question: SurveyQuestion | undefined,
): question is SurveyQuestion & { id: string } {
return (
hasQuestionIdentity(question) &&
question.type === "rating" &&
question.display === "emoji" &&
question.scale === 2 &&
question.optional !== true
);
}
function isReasonsQuestion(
question: SurveyQuestion | undefined,
): question is SurveyQuestion & { id: string } {
return (
hasQuestionIdentity(question) &&
question.type === "multiple_choice" &&
question.optional === true &&
Array.isArray(question.choices) &&
question.choices.length === LIGHTHOUSE_FEEDBACK_REASONS.length &&
question.choices.every(
(choice, index) => choice === LIGHTHOUSE_FEEDBACK_REASONS[index],
)
);
}
function isDetailsQuestion(
question: SurveyQuestion | undefined,
): question is SurveyQuestion & { id: string } {
return (
hasQuestionIdentity(question) &&
question.type === "open" &&
question.optional === true
);
}
function hasQuestionIdentity(
question: SurveyQuestion | undefined,
): question is SurveyQuestion & { id: string } {
return (
typeof question?.id === "string" &&
question.id.trim().length > 0 &&
typeof question.question === "string" &&
question.question.length > 0
);
}
function toFeedbackQuestion(question: SurveyQuestion & { id: string }) {
return { id: question.id, question: question.question };
}
ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-page.test.tsx
+238
View File
@@ -1,5 +1,6 @@
import { act, render, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import type { Survey } from "posthog-js";
import { type ReactNode } from "react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
@@ -34,11 +35,15 @@ const {
getMessagesMock,
sendMessageMock,
updateConfigurationMock,
captureMock,
onSurveysLoadedMock,
} = vi.hoisted(() => ({
createSessionMock: vi.fn(),
getMessagesMock: vi.fn(),
sendMessageMock: vi.fn(),
updateConfigurationMock: vi.fn(),
captureMock: vi.fn(),
onSurveysLoadedMock: vi.fn(),
}));
vi.mock("@/app/(prowler)/lighthouse/_actions", () => ({
@@ -48,6 +53,13 @@ vi.mock("@/app/(prowler)/lighthouse/_actions", () => ({
updateLighthouseV2Configuration: updateConfigurationMock,
}));
vi.mock("posthog-js", () => ({
default: {
capture: captureMock,
onSurveysLoaded: onSurveysLoadedMock,
},
}));
vi.mock("next/navigation", () => ({
usePathname: () => window.location.pathname,
useSearchParams: () => new URLSearchParams(window.location.search),
@@ -97,6 +109,43 @@ const supportedProviders: LighthouseV2SupportedProvider[] = [
{ id: "openai-compatible", name: "OpenAI Compatible" },
];
const OUTCOME_FEEDBACK_SURVEY = {
id: "survey-123",
name: "Lighthouse Request Outcome Feedback",
type: "api",
questions: [
{
id: "rating-question-id",
type: "rating",
display: "emoji",
scale: 2,
question: "How was this outcome?",
lowerBoundLabel: "Not helpful",
upperBoundLabel: "Helpful",
},
{
id: "reasons-question-id",
type: "multiple_choice",
optional: true,
choices: [
"Don't like the style",
"Didn't fully follow instructions",
"Low quality",
"Biased",
"Safety or legal concern",
"Other",
],
question: "What could be improved?",
},
{
id: "details-question-id",
type: "open",
optional: true,
question: "Additional feedback",
},
],
} as unknown as Survey;
describe("LighthouseV2ChatPage", () => {
beforeEach(() => {
vi.stubGlobal(
@@ -115,6 +164,12 @@ describe("LighthouseV2ChatPage", () => {
getMessagesMock.mockReset();
sendMessageMock.mockReset();
updateConfigurationMock.mockReset();
captureMock.mockReset();
onSurveysLoadedMock.mockReset();
onSurveysLoadedMock.mockImplementation((callback) => {
callback([OUTCOME_FEEDBACK_SURVEY]);
return () => {};
});
resetPanelChatStoreForTests();
eventSources = stubEventSource();
window.history.replaceState(null, "", "/lighthouse");
@@ -545,6 +600,156 @@ describe("LighthouseV2ChatPage", () => {
);
});
it("renders each outcome control under its paired assistant answer and submits against the initiating user message", async () => {
// Given
const user = userEvent.setup();
const firstUserMessage = message("message-user-1", "user", "First prompt");
const secondUserMessage = message(
"message-user-2",
"user",
"Second prompt",
);
renderPage({
initialSessionId: "session-1",
initialMessages: [
firstUserMessage,
message("message-assistant-1", "assistant", "First answer"),
secondUserMessage,
message("message-assistant-2", "assistant", "Second answer"),
],
});
const firstPrompt = screen.getByText("First prompt").closest("article");
const firstAnswer = screen.getByText("First answer").closest("article");
const secondPrompt = screen.getByText("Second prompt").closest("article");
const secondAnswer = screen.getByText("Second answer").closest("article");
if (!firstPrompt || !firstAnswer || !secondPrompt || !secondAnswer) {
throw new Error("Expected every chat message to render");
}
// When
await user.click(
within(firstAnswer).getByRole("button", {
name: "Mark outcome as not helpful",
}),
);
await user.click(screen.getByRole("button", { name: "Submit" }));
await user.click(
within(secondAnswer).getByRole("button", {
name: "Mark outcome as not helpful",
}),
);
await user.click(screen.getByRole("button", { name: "Submit" }));
// Then
expect(
within(firstPrompt).queryByRole("button", { name: /Mark outcome/ }),
).not.toBeInTheDocument();
expect(
within(secondPrompt).queryByRole("button", { name: /Mark outcome/ }),
).not.toBeInTheDocument();
expect(
within(firstAnswer).getByRole("button", {
name: "Mark outcome as helpful",
}),
).toBeInTheDocument();
expect(
within(secondAnswer).getByRole("button", {
name: "Mark outcome as helpful",
}),
).toBeInTheDocument();
expect(captureMock).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
$ai_trace_id: "message-user-1",
"$survey_response_rating-question-id": 2,
}),
);
expect(captureMock).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
$ai_trace_id: "message-user-2",
"$survey_response_rating-question-id": 2,
}),
);
expect(captureMock).toHaveBeenCalledTimes(2);
expect(onSurveysLoadedMock).toHaveBeenCalledOnce();
});
it.each([
["unavailable", []],
[
"invalid",
[
{
...OUTCOME_FEEDBACK_SURVEY,
questions: [
{ ...OUTCOME_FEEDBACK_SURVEY.questions[0], optional: true },
OUTCOME_FEEDBACK_SURVEY.questions[1],
OUTCOME_FEEDBACK_SURVEY.questions[2],
],
} as unknown as Survey,
],
],
])(
"does not render outcome feedback controls when the API survey is %s",
(_state, surveys) => {
// Given
onSurveysLoadedMock.mockImplementation((callback) => {
callback(surveys);
return () => {};
});
// When
renderPage({
initialSessionId: "session-1",
initialMessages: [
message("message-user-1", "user", "Prompt"),
message("message-assistant-1", "assistant", "Answer"),
],
});
// Then
expect(
screen.queryByRole("button", { name: "Mark outcome as helpful" }),
).not.toBeInTheDocument();
expect(
screen.queryByRole("button", { name: "Mark outcome as not helpful" }),
).not.toBeInTheDocument();
},
);
it("cleans up the outcome feedback survey subscription when the chat unmounts", () => {
// Given
const unsubscribe = vi.fn();
onSurveysLoadedMock.mockImplementation((callback) => {
callback([OUTCOME_FEEDBACK_SURVEY]);
return unsubscribe;
});
const { unmount } = renderPage();
// When
unmount();
// Then
expect(unsubscribe).toHaveBeenCalledOnce();
});
it("does not attach outcome feedback to an assistant after an optimistic user prompt", () => {
// Given / When
renderPage({
initialSessionId: "session-1",
initialMessages: [
message("optimistic-user-1", "user", "Pending prompt"),
message("message-assistant-1", "assistant", "Unpaired answer"),
],
});
// Then
expect(
screen.queryByRole("button", { name: /Mark outcome/ }),
).not.toBeInTheDocument();
});
it("hands the suggested follow-up skill off to a fresh session", async () => {
// Given: a persisted triage run in the currently open session
const user = userEvent.setup();
@@ -698,6 +903,39 @@ describe("LighthouseV2ChatPage", () => {
expect(sendMessageMock).toHaveBeenCalledTimes(1);
});
it("keeps outcome feedback available after a terminal run error", async () => {
// Given
const user = userEvent.setup();
getMessagesMock.mockResolvedValue({
data: [message("task-1", "user", "Run this check")],
});
renderPage();
await user.type(
screen.getByRole("textbox", { name: "Message" }),
["Run this check", "{Enter}"].join(""),
);
await waitFor(() => expect(eventSources).toHaveLength(1));
// When
act(() => eventSources[0].emit("error", { detail: "Agent run failed." }));
// Then
expect(await screen.findByText("Agent run failed.")).toBeInTheDocument();
await user.click(
await screen.findByRole("button", {
name: "Mark outcome as not helpful",
}),
);
await user.click(screen.getByRole("button", { name: "Submit" }));
expect(captureMock).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
$ai_trace_id: "task-1",
"$survey_response_rating-question-id": 2,
}),
);
});
it("surfaces a connection error when the stream closes without retrying", async () => {
// Given
const user = userEvent.setup();
ui/app/(prowler)/lighthouse/_components/chat/lighthouse-v2-chat-view.tsx
+39 -5
View File
@@ -1,6 +1,7 @@
"use client";
import { type ReactNode, type SubmitEvent } from "react";
import posthogClient from "posthog-js";
import { type ReactNode, type SubmitEvent, useState } from "react";
import {
Conversation,
@@ -19,6 +20,7 @@ import {
parseLighthouseV2ModelSelectionValue,
} from "@/app/(prowler)/lighthouse/_lib/model-selection";
import {
LIGHTHOUSE_V2_MESSAGE_ROLE,
LIGHTHOUSE_V2_PROVIDER_TYPE,
type LighthouseV2Configuration,
type LighthouseV2ProviderType,
@@ -33,12 +35,17 @@ import {
} from "@/components/shadcn/combobox/combobox";
import { Skeleton } from "@/components/shadcn/skeleton/skeleton";
import { useLighthouseCurrentContext } from "@/hooks/use-lighthouse-context";
import { useMountEffect } from "@/hooks/use-mount-effect";
import { ProviderIcon } from "../config/provider-icon";
import { ChatComposerPanel } from "./composer";
import { ChatEmptyState } from "./empty-state";
import { useLighthouseChatStore } from "./lighthouse-chat-store-provider";
import {
resolveLighthouseFeedbackSurvey,
type LighthouseFeedbackSurvey,
} from "./lighthouse-feedback-survey";
import { MessageBubble } from "./message-bubble";
import { SkillComposerPill } from "./skill-composer-pill";
import { SkillRunProgress } from "./skill-run-progress";
@@ -62,6 +69,7 @@ export function LighthouseV2ChatView({
emptyStateFooter,
}: LighthouseV2ChatViewProps) {
const currentContext = useLighthouseCurrentContext();
const feedbackSurvey = useLighthouseOutcomeFeedbackSurvey();
// Whole-store subscription is intentional: the view renders most of the state and selectLighthouseChatCanSend takes full state.
const state = useLighthouseChatStore((current) => current);
const {
@@ -72,6 +80,7 @@ export function LighthouseV2ChatView({
feedback,
isLoadingSession,
lastSubmission,
failedOutcomeMessageId,
selectedModelSelection,
modelPreferenceSaving,
setInput,
@@ -142,9 +151,14 @@ export function LighthouseV2ChatView({
Boolean(streamState.assistantText) ||
streamState.toolCalls.length > 0;
const hasConversation = messages.length > 0 || hasLiveAssistantActivity;
const failedOutcomeFeedbackTarget = failedOutcomeMessageId
? messages.find((message) => message.id === failedOutcomeMessageId)
: undefined;
const composerPanelProps = {
feedback,
feedbackTarget: failedOutcomeFeedbackTarget,
feedbackSurvey,
canRetry:
streamState.status === LIGHTHOUSE_V2_STREAM_STATUS.DISCONNECTED &&
lastSubmission !== null,
@@ -200,14 +214,22 @@ export function LighthouseV2ChatView({
scrollClassName="minimal-scrollbar overflow-x-hidden overflow-y-auto"
>
{messages.map((message, index) => {
const skillRun = getSkillRunFromLaunch(
message,
messages[index - 1],
);
const previousMessage = messages[index - 1];
const skillRun = getSkillRunFromLaunch(message, previousMessage);
// The assistant owns the controls visually; its adjacent persisted
// user turn remains the API task/trace feedback target.
const feedbackTarget =
message.role === LIGHTHOUSE_V2_MESSAGE_ROLE.ASSISTANT &&
previousMessage?.role === LIGHTHOUSE_V2_MESSAGE_ROLE.USER &&
!previousMessage.id.startsWith("optimistic-")
? previousMessage
: undefined;
return (
<MessageBubble
key={message.id}
message={message}
feedbackTarget={feedbackTarget}
feedbackSurvey={feedbackSurvey}
skillRun={skillRun}
onLaunchSkill={(skill) => {
// The DyR prompts hand follow-up skills off to a separate
@@ -275,6 +297,18 @@ export function LighthouseV2ChatView({
);
}
function useLighthouseOutcomeFeedbackSurvey(): LighthouseFeedbackSurvey | null {
const [survey, setSurvey] = useState<LighthouseFeedbackSurvey | null>(null);
useMountEffect(() => {
return posthogClient.onSurveysLoaded((surveys) => {
setSurvey(resolveLighthouseFeedbackSurvey(surveys));
});
});
return survey;
}
function SessionLoadingState() {
return (
<div
ui/app/(prowler)/lighthouse/_components/chat/message-bubble.test.tsx
+399 -2
View File
@@ -1,7 +1,7 @@
import { render, screen } from "@testing-library/react";
import { render, screen, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { type ReactNode } from "react";
import { describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
LIGHTHOUSE_V2_MESSAGE_ROLE,
@@ -9,8 +9,37 @@ import {
type LighthouseV2Message,
} from "@/app/(prowler)/lighthouse/_types";
import {
LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH,
type LighthouseFeedbackSurvey,
} from "./lighthouse-feedback-survey";
import { MessageBubble } from "./message-bubble";
const { captureMock } = vi.hoisted(() => ({
captureMock: vi.fn(),
}));
vi.mock("posthog-js", () => ({
default: { capture: captureMock },
}));
const FEEDBACK_SURVEY = {
id: "survey-123",
name: "Lighthouse Request Outcome Feedback",
ratingQuestion: {
id: "rating-question-id",
question: "How was this outcome?",
},
reasonsQuestion: {
id: "reasons-question-id",
question: "What could be improved?",
},
detailsQuestion: {
id: "details-question-id",
question: "Additional feedback",
},
} satisfies LighthouseFeedbackSurvey;
vi.mock("streamdown", () => ({
Streamdown: ({ children }: { children: ReactNode }) => {
const text = String(children);
@@ -48,6 +77,14 @@ vi.mock("streamdown", () => ({
}));
describe("MessageBubble", () => {
beforeEach(() => {
captureMock.mockReset();
});
afterEach(() => {
vi.restoreAllMocks();
});
it("should never render the agent-facing context block for user messages", () => {
// Given
const userMessage: LighthouseV2Message = {
@@ -341,6 +378,345 @@ describe("MessageBubble", () => {
expect(isBefore(firstText, toolCall)).toBe(true);
expect(isBefore(toolCall, secondText)).toBe(true);
});
describe("when rendering feedback controls", () => {
it("should show controls for an assistant answer with a persisted user feedback target", () => {
// Given / When
renderFeedbackBubble();
// Then
expect(
screen.getByRole("button", { name: "Mark outcome as helpful" }),
).toBeInTheDocument();
expect(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
).toBeInTheDocument();
});
it("should not show controls under a user prompt", () => {
// Given
const message = buildUserMessage();
// When
render(<MessageBubble message={message} />);
// Then
expect(
screen.queryByRole("button", { name: "Mark outcome as helpful" }),
).not.toBeInTheDocument();
});
it("should not show controls for an assistant without a feedback target", () => {
// Given
const message = buildAssistantMessage([textPart("part-1", "Done")]);
// When
render(<MessageBubble message={message} />);
// Then
expect(
screen.queryByRole("button", { name: "Mark outcome as helpful" }),
).not.toBeInTheDocument();
});
it("should not show controls for an optimistic user feedback target", () => {
// Given
const message = buildAssistantMessage([textPart("part-1", "Done")]);
// When
render(
<MessageBubble
message={message}
feedbackTarget={buildUserMessage("optimistic-user-1")}
feedbackSurvey={FEEDBACK_SURVEY}
/>,
);
// Then
expect(
screen.queryByRole("button", { name: "Mark outcome as helpful" }),
).not.toBeInTheDocument();
});
});
describe("when rating an assistant answer", () => {
it("should submit thumbs up immediately without opening the feedback form", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
// When
await user.click(
screen.getByRole("button", { name: "Mark outcome as helpful" }),
);
// Then
expect(captureMock).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
$ai_trace_id: "message-user-1",
$survey_id: "survey-123",
$survey_name: "Lighthouse Request Outcome Feedback",
"$survey_response_rating-question-id": 1,
$survey_completed: true,
}),
);
expect(
screen.queryByRole("heading", { name: "Share feedback" }),
).not.toBeInTheDocument();
});
it("should keep controls usable and assign a fresh submission identifier for each capture", async () => {
// Given
const user = userEvent.setup();
vi.spyOn(globalThis.crypto, "randomUUID")
.mockReturnValueOnce("submission-1")
.mockReturnValueOnce("submission-2");
renderFeedbackBubble();
const helpfulButton = screen.getByRole("button", {
name: "Mark outcome as helpful",
});
// When
await user.click(helpfulButton);
// Then
expect(helpfulButton).toBeEnabled();
expect(captureMock).toHaveBeenNthCalledWith(
1,
"survey sent",
expect.objectContaining({
$survey_submission_id: "submission-1",
}),
);
// When
await user.click(helpfulButton);
// Then
expect(captureMock).toHaveBeenNthCalledWith(
2,
"survey sent",
expect.objectContaining({
$survey_submission_id: "submission-2",
}),
);
});
it("should open the feedback form with the chosen rating without submitting", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
// When
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
// Then
expect(
screen.getByRole("heading", { name: "Share feedback" }),
).toBeVisible();
expect(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
).toHaveAttribute("aria-pressed", "true");
expect(
screen.getByLabelText("Additional feedback (optional)"),
).toHaveAttribute(
"maxlength",
String(LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH),
);
expect(captureMock).not.toHaveBeenCalled();
});
it("should submit selected feedback reasons with trimmed optional details", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
// When
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
const styleReason = screen.getByRole("button", {
name: "Don't like the style",
});
expect(
within(screen.getByRole("group", { name: "Reasons (optional)" }))
.getAllByRole("button")
.map((button) => button.textContent),
).toEqual([
"Don't like the style",
"Didn't fully follow instructions",
"Low quality",
"Biased",
"Safety or legal concern",
"Other",
]);
styleReason.focus();
await user.keyboard("{Enter}");
await user.click(screen.getByRole("button", { name: "Low quality" }));
// Then - keyboard and pointer interactions retain a multi-select pressed state.
expect(styleReason).toHaveAttribute("aria-pressed", "true");
expect(
screen.getByRole("button", { name: "Low quality" }),
).toHaveAttribute("aria-pressed", "true");
// When
await user.type(
screen.getByLabelText("Additional feedback (optional)"),
" Missing evidence ",
);
await user.click(screen.getByRole("button", { name: "Submit" }));
// Then
expect(captureMock).toHaveBeenCalledTimes(3);
expect(captureMock).toHaveBeenLastCalledWith(
"survey sent",
expect.objectContaining({
$ai_trace_id: "message-user-1",
"$survey_response_rating-question-id": 2,
"$survey_response_reasons-question-id": [
"Don't like the style",
"Low quality",
],
"$survey_response_details-question-id": "Missing evidence",
$survey_completed: true,
}),
);
});
it("should submit thumbs down without optional reasons or details", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
// When
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
await user.click(screen.getByRole("button", { name: "Submit" }));
// Then
expect(captureMock).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
$ai_trace_id: "message-user-1",
"$survey_response_rating-question-id": 2,
$survey_completed: true,
}),
);
});
it("should cancel without submitting and clear the draft", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
await user.click(
screen.getByRole("button", { name: "Don't like the style" }),
);
await user.type(
screen.getByLabelText("Additional feedback (optional)"),
"Unsaved draft",
);
// When
await user.click(screen.getByRole("button", { name: "Cancel" }));
// Then
expect(captureMock).not.toHaveBeenCalled();
expect(
screen.queryByRole("heading", { name: "Share feedback" }),
).not.toBeInTheDocument();
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
expect(
screen.getByLabelText("Additional feedback (optional)"),
).toHaveValue("");
expect(
screen.getByRole("button", { name: "Don't like the style" }),
).toHaveAttribute("aria-pressed", "false");
});
it("should clear selected feedback reasons after a successful submission", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
await user.click(
screen.getByRole("button", { name: "Don't like the style" }),
);
// When
await user.click(screen.getByRole("button", { name: "Submit" }));
expect(
screen.queryByRole("heading", { name: "Share feedback" }),
).not.toBeInTheDocument();
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
// Then
expect(
screen.getByRole("button", { name: "Don't like the style" }),
).toHaveAttribute("aria-pressed", "false");
});
it("should close the feedback popup without submitting", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
// When
await user.keyboard("{Escape}");
// Then
expect(captureMock).not.toHaveBeenCalled();
expect(
screen.queryByRole("heading", { name: "Share feedback" }),
).not.toBeInTheDocument();
});
it("should discard the feedback draft when the popup closes with Escape", async () => {
// Given
const user = userEvent.setup();
renderFeedbackBubble();
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
await user.click(
screen.getByRole("button", { name: "Don't like the style" }),
);
await user.type(
screen.getByLabelText("Additional feedback (optional)"),
"Unsaved draft",
);
// When
await user.keyboard("{Escape}");
await user.click(
screen.getByRole("button", { name: "Mark outcome as not helpful" }),
);
// Then
expect(
screen.getByLabelText("Additional feedback (optional)"),
).toHaveValue("");
expect(
screen.getByRole("button", { name: "Don't like the style" }),
).toHaveAttribute("aria-pressed", "false");
});
});
});
function isBefore(first: HTMLElement, second: HTMLElement): boolean {
@@ -362,6 +738,27 @@ function buildAssistantMessage(
};
}
function buildUserMessage(id = "message-user-1"): LighthouseV2Message {
return {
id,
role: LIGHTHOUSE_V2_MESSAGE_ROLE.USER,
model: null,
tokenUsage: null,
insertedAt: "2026-06-25T10:00:00Z",
parts: [textPart("part-user-1", "Run this check")],
};
}
function renderFeedbackBubble() {
return render(
<MessageBubble
message={buildAssistantMessage([textPart("part-1", "Done")])}
feedbackTarget={buildUserMessage()}
feedbackSurvey={FEEDBACK_SURVEY}
/>,
);
}
function textPart(
id: string,
text: string,
ui/app/(prowler)/lighthouse/_components/chat/message-bubble.tsx
+205 -7
View File
@@ -1,6 +1,14 @@
"use client";
import { Bot, Check, Copy, UserRound } from "lucide-react";
import {
Bot,
Check,
Copy,
ThumbsDown,
ThumbsUp,
UserRound,
} from "lucide-react";
import posthogClient from "posthog-js";
import { useState } from "react";
import { formatMessageTimestamp } from "@/app/(prowler)/lighthouse/_lib/format";
@@ -18,9 +26,24 @@ import {
} from "@/app/(prowler)/lighthouse/_types";
import { LighthouseContextBadge } from "@/components/lighthouse/context-chip";
import { Button } from "@/components/shadcn/button/button";
import {
Popover,
PopoverAnchor,
PopoverContent,
} from "@/components/shadcn/popover";
import { FeedbackForm } from "@/components/survey/feedback-form";
import { cn } from "@/lib/utils";
import type { LighthouseSkillDefinition } from "@/types/lighthouse-skills";
import {
buildLighthouseFeedbackSurveyEvents,
LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH,
LIGHTHOUSE_FEEDBACK_RATING,
LIGHTHOUSE_FEEDBACK_REASON,
type LighthouseFeedbackRating,
type LighthouseFeedbackReason,
type LighthouseFeedbackSurvey,
} from "./lighthouse-feedback-survey";
import { MessageMarkdown } from "./message-markdown";
import { SkillActionsRow, SkillRunReceipt } from "./skill-completed";
import { SkillMessageCard } from "./skill-message-card";
@@ -34,22 +57,32 @@ const ASSISTANT_PART_GROUP_TYPE = {
type AssistantPartGroupType =
(typeof ASSISTANT_PART_GROUP_TYPE)[keyof typeof ASSISTANT_PART_GROUP_TYPE];
const LIGHTHOUSE_FEEDBACK_REASON_OPTIONS = Object.values(
LIGHTHOUSE_FEEDBACK_REASON,
).map((reason) => ({ value: reason, label: reason }));
interface AssistantPartGroup {
id: string;
type: AssistantPartGroupType;
parts: LighthouseV2Part[];
}
export function MessageBubble({
message,
skillRun,
onLaunchSkill,
}: {
interface MessageBubbleProps {
message: LighthouseV2Message;
feedbackTarget?: LighthouseV2Message;
feedbackSurvey?: LighthouseFeedbackSurvey | null;
// Present when this assistant message answered a skill launch (design 1j).
skillRun?: SkillRunInfo;
onLaunchSkill?: (skill: LighthouseSkillDefinition) => void;
}) {
}
export function MessageBubble({
message,
feedbackTarget,
feedbackSurvey,
skillRun,
onLaunchSkill,
}: MessageBubbleProps) {
const isUser = message.role === LIGHTHOUSE_V2_MESSAGE_ROLE.USER;
const isSkillResponse = !isUser && skillRun !== undefined;
// Text-only join feeds the copy button; tool calls are rendered separately.
@@ -125,6 +158,8 @@ export function MessageBubble({
isUser={isUser}
text={messageText}
insertedAt={message.insertedAt}
feedbackTarget={feedbackTarget}
feedbackSurvey={feedbackSurvey}
/>
</div>
{isUser && (
@@ -204,10 +239,14 @@ function MessageMeta({
isUser,
text,
insertedAt,
feedbackTarget,
feedbackSurvey,
}: {
isUser: boolean;
text: string;
insertedAt: string;
feedbackTarget?: LighthouseV2Message;
feedbackSurvey?: LighthouseFeedbackSurvey | null;
}) {
// Copy is always shown; the timestamp only reveals on hover over the message.
// Agent footer reads left-to-right ([copy] [time]); user footer mirrors it.
@@ -219,6 +258,13 @@ function MessageMeta({
)}
>
<CopyMessageButton text={text} />
{feedbackTarget && feedbackSurvey && (
<LighthouseOutcomeFeedbackControls
key={feedbackTarget.id}
message={feedbackTarget}
survey={feedbackSurvey}
/>
)}
<time
dateTime={insertedAt}
className="text-text-neutral-tertiary text-xs opacity-0 transition-opacity group-hover:opacity-100"
@@ -229,6 +275,158 @@ function MessageMeta({
);
}
export function LighthouseOutcomeFeedbackControls({
message,
survey,
}: {
message: LighthouseV2Message;
survey: LighthouseFeedbackSurvey;
}) {
const [open, setOpen] = useState(false);
const [rating, setRating] = useState<LighthouseFeedbackRating | null>(null);
// Local state needed: reasons and details are buffered until "Submit" is clicked.
const [reasons, setReasons] = useState<LighthouseFeedbackReason[]>([]);
const [details, setDetails] = useState("");
const isPersistedUserMessage =
message.role === LIGHTHOUSE_V2_MESSAGE_ROLE.USER &&
!message.id.startsWith("optimistic-");
if (!isPersistedUserMessage) return null;
const submit = (
submittedRating = rating,
submittedDetails = details,
submittedReasons = reasons,
) => {
if (!submittedRating) return;
const submissionId = globalThis.crypto?.randomUUID();
if (!submissionId) return;
for (const event of buildLighthouseFeedbackSurveyEvents(survey, {
targetMessageId: message.id,
rating: submittedRating,
reasons: submittedReasons,
details: submittedDetails,
submissionId,
})) {
posthogClient.capture("survey sent", event);
}
setOpen(false);
setReasons([]);
setDetails("");
};
const selectRating = (nextRating: LighthouseFeedbackRating) => {
setRating(nextRating);
if (nextRating === LIGHTHOUSE_FEEDBACK_RATING.UP) {
submit(nextRating, "", []);
return;
}
setOpen(true);
};
const toggleReason = (reason: LighthouseFeedbackReason) => {
setReasons((current) =>
current.includes(reason)
? current.filter((currentReason) => currentReason !== reason)
: [...current, reason],
);
};
const cancel = () => {
setOpen(false);
setRating(null);
setReasons([]);
setDetails("");
};
const handleOpenChange = (nextOpen: boolean) => {
if (nextOpen) {
setOpen(true);
return;
}
cancel();
};
return (
<Popover open={open} onOpenChange={handleOpenChange}>
<PopoverAnchor asChild>
<div className="flex items-center gap-0.5">
<FeedbackRatingButton
rating={LIGHTHOUSE_FEEDBACK_RATING.UP}
selectedRating={rating}
onSelect={selectRating}
/>
<FeedbackRatingButton
rating={LIGHTHOUSE_FEEDBACK_RATING.DOWN}
selectedRating={rating}
onSelect={selectRating}
/>
</div>
</PopoverAnchor>
<PopoverContent
align="start"
side="top"
className="w-[min(92vw,26rem)] p-5"
>
<FeedbackForm
title="Share feedback"
description="Tell us more about this answer."
reasons={{
label: "Reasons (optional)",
options: LIGHTHOUSE_FEEDBACK_REASON_OPTIONS,
selected: reasons,
onToggle: toggleReason,
}}
detailsLabel="Additional feedback (optional)"
placeholder="Type your answer here"
details={details}
detailsMaxLength={LIGHTHOUSE_FEEDBACK_DETAILS_MAX_LENGTH}
onDetailsChange={setDetails}
onSubmit={submit}
onCancel={cancel}
/>
</PopoverContent>
</Popover>
);
}
function FeedbackRatingButton({
rating,
selectedRating,
onSelect,
}: {
rating: LighthouseFeedbackRating;
selectedRating: LighthouseFeedbackRating | null;
onSelect: (rating: LighthouseFeedbackRating) => void;
}) {
const isUp = rating === LIGHTHOUSE_FEEDBACK_RATING.UP;
const selected = selectedRating === rating;
const Icon = isUp ? ThumbsUp : ThumbsDown;
return (
<Button
type="button"
variant="ghost"
size="icon-sm"
aria-label={
isUp ? "Mark outcome as helpful" : "Mark outcome as not helpful"
}
aria-pressed={selected}
onClick={() => onSelect(rating)}
className={cn(
"text-text-neutral-tertiary hover:text-text-neutral-primary size-6",
selected &&
"bg-button-primary hover:bg-button-primary-hover active:bg-button-primary-press focus-visible:ring-button-primary/50 text-black hover:text-black",
)}
>
<Icon className="size-3.5" />
</Button>
);
}
function CopyMessageButton({ text }: { text: string }) {
const [copied, setCopied] = useState(false);
ui/app/(prowler)/lighthouse/_lib/chat-store.test.ts
+91
View File
@@ -351,6 +351,97 @@ describe("createLighthouseChatStore", () => {
expect(store.getState().streamState.activeTaskId).toBeNull();
});
it("keeps the persisted user message ratable after a terminal error", async () => {
// Given
const store = makeStore();
await store.getState().submitMessage("Run this check");
getMessagesMock.mockResolvedValue({
data: [message("task-1", "user", "Run this check")],
});
// When
eventSources[0].emit("error", { detail: "Agent run failed." });
// Then
await vi.waitFor(() =>
expect(store.getState().messages).toEqual([
expect.objectContaining({
id: "task-1",
role: "user",
}),
]),
);
expect(store.getState().feedback).toBe("Agent run failed.");
expect(store.getState().failedOutcomeMessageId).toBe("task-1");
});
it("keeps a terminal error ratable when it arrives before the send resolves", async () => {
// Given
const store = makeStore();
let resolveSend: (value: unknown) => void = () => {};
sendMessageMock.mockReturnValueOnce(
new Promise((resolve) => {
resolveSend = resolve;
}),
);
getMessagesMock.mockResolvedValue({
data: [message("task-1", "user", "Run this check")],
});
const submitting = store.getState().submitMessage("Run this check");
await vi.waitFor(() => expect(sendMessageMock).toHaveBeenCalledOnce());
// When
eventSources[0].emit("error", { detail: "Agent run failed." });
await vi.waitFor(() =>
expect(store.getState().feedback).toBe("Agent run failed."),
);
resolveSend({
data: {
task: { id: "task-1", name: "lighthouse-run", state: "executing" },
},
});
await submitting;
// Then
await vi.waitFor(() =>
expect(store.getState().failedOutcomeMessageId).toBe("task-1"),
);
expect(store.getState().messages).toEqual([
expect.objectContaining({ id: "task-1", role: "user" }),
]);
});
it("reloads a completed message when it arrives before the send resolves", async () => {
// Given
const store = makeStore();
let resolveSend: (value: unknown) => void = () => {};
sendMessageMock.mockReturnValueOnce(
new Promise((resolve) => {
resolveSend = resolve;
}),
);
getMessagesMock.mockResolvedValue({
data: [message("message-1", "assistant", "Persisted answer")],
});
const submitting = store.getState().submitMessage("Summarize findings");
await vi.waitFor(() => expect(sendMessageMock).toHaveBeenCalledOnce());
// When
eventSources[0].emit("message.end", { message_id: "message-1" });
resolveSend({
data: {
task: { id: "task-1", name: "lighthouse-run", state: "executing" },
},
});
await submitting;
// Then
expect(store.getState().messages).toEqual([
expect.objectContaining({ id: "message-1", role: "assistant" }),
]);
expect(store.getState().failedOutcomeMessageId).toBeNull();
});
it("blocks sending and refreshes messages on a 409 conflict", async () => {
// Given
const store = makeStore();
ui/app/(prowler)/lighthouse/_lib/chat-store.ts
+102 -24
View File
@@ -21,6 +21,7 @@ import { notifyLighthouseV2SessionsChanged } from "@/app/(prowler)/lighthouse/_l
import { parseStreamEvent } from "@/app/(prowler)/lighthouse/_lib/stream-event-parser";
import { buildLighthouseV2StreamUrl } from "@/app/(prowler)/lighthouse/_lib/stream-url";
import {
LIGHTHOUSE_V2_MESSAGE_ROLE,
LIGHTHOUSE_V2_PROVIDER_TYPE,
LIGHTHOUSE_V2_SSE_EVENT,
type LighthouseV2Configuration,
@@ -65,6 +66,7 @@ export interface LighthouseChatState {
isSubmitting: boolean;
isLoadingSession: boolean;
lastSubmission: LighthouseChatSubmission | null;
failedOutcomeMessageId: string | null;
selectedModelSelection: LighthouseV2ModelSelection | null;
modelPreferenceSaving: boolean;
setSessionUrlSyncEnabled: (enabled: boolean) => void;
@@ -89,8 +91,29 @@ export interface LighthouseChatSubmission {
skill?: LighthouseSkillDefinition;
}
interface LighthouseChatPendingTerminalEvent {
sessionId: string;
event: LighthouseV2SSEEvent;
submissionVersion: number;
}
export type LighthouseChatStore = StoreApi<LighthouseChatState>;
function findLastPersistedUserMessageId(
messages: LighthouseV2Message[],
): string | null {
for (let index = messages.length - 1; index >= 0; index -= 1) {
const message = messages[index];
if (
message?.role === LIGHTHOUSE_V2_MESSAGE_ROLE.USER &&
!message.id.startsWith("optimistic-")
) {
return message.id;
}
}
return null;
}
export function selectLighthouseChatCanSend(
state: LighthouseChatState,
): boolean {
@@ -142,6 +165,7 @@ export function createLighthouseChatStore(
// Each accepted submit owns its loading state. A reset can start a
// replacement while the cancelled submit is still settling.
let submissionIntentVersion = 0;
let pendingTerminalEvent: LighthouseChatPendingTerminalEvent | null = null;
let syncUrlToSession = options.syncUrlToSession;
const syncSessionUrl = (sessionId: string | null) => {
@@ -174,41 +198,67 @@ export function createLighthouseChatStore(
return false;
};
const handleTerminalEvent = async (
const refreshAfterTerminalEvent = async (
sessionId: string,
event: LighthouseV2SSEEvent,
) => {
if (
event.type === LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_END ||
event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR
) {
closeStream();
set({ blockedByConflict: false });
if (event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR) {
set({ feedback: event.detail || "Agent run failed." });
}
// A fast follow-up can start while this refresh is in flight; applying
// it would erase the new optimistic message and provisional task id.
const noNewerSubmission = () =>
!get().isSubmitting && !get().streamState.activeTaskId;
const refreshed = await refreshMessages(sessionId, noNewerSubmission);
if (refreshed) {
set({ streamState: createInitialLighthouseV2StreamState() });
}
notifyLighthouseV2SessionsChanged();
// A fast follow-up can start while this refresh is in flight; applying
// it would erase the new optimistic message and provisional task id.
const noNewerSubmission = () =>
!get().isSubmitting && !get().streamState.activeTaskId;
const refreshed = await refreshMessages(sessionId, noNewerSubmission);
if (refreshed) {
const failedOutcomeMessageId =
event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR
? findLastPersistedUserMessageId(get().messages)
: null;
set({
streamState: createInitialLighthouseV2StreamState(),
failedOutcomeMessageId,
});
}
notifyLighthouseV2SessionsChanged();
};
const startStream = (streamUrl: string, sessionId: string) => {
const handleTerminalEvent = async (
sessionId: string,
event: LighthouseV2SSEEvent,
submissionVersion: number,
) => {
if (
event.type !== LIGHTHOUSE_V2_SSE_EVENT.MESSAGE_END &&
event.type !== LIGHTHOUSE_V2_SSE_EVENT.ERROR
) {
return;
}
closeStream();
set({ blockedByConflict: false });
if (event.type === LIGHTHOUSE_V2_SSE_EVENT.ERROR) {
set({ feedback: event.detail || "Agent run failed." });
}
if (get().isSubmitting) {
pendingTerminalEvent = { sessionId, event, submissionVersion };
return;
}
await refreshAfterTerminalEvent(sessionId, event);
};
const startStream = (
streamUrl: string,
sessionId: string,
submissionVersion: number,
) => {
closeStream();
const source = new EventSource(streamUrl);
eventSource = source;
const applyEvent = (event: LighthouseV2SSEEvent) => {
if (submissionVersion !== submissionIntentVersion) return;
set((current) => ({
streamState: reduceLighthouseV2Event(current.streamState, event),
}));
void handleTerminalEvent(sessionId, event);
void handleTerminalEvent(sessionId, event, submissionVersion);
};
source.addEventListener("message.delta", (event) =>
@@ -313,6 +363,7 @@ export function createLighthouseChatStore(
};
set((current) => ({
feedback: null,
failedOutcomeMessageId: null,
blockedByConflict: false,
lastSubmission,
input: "",
@@ -326,7 +377,11 @@ export function createLighthouseChatStore(
// Subscribe to the same-origin SSE proxy BEFORE sending the message:
// the backend has no replay buffer, so the listener must be attached
// before the worker starts emitting.
startStream(buildLighthouseV2StreamUrl(sessionId), sessionId);
startStream(
buildLighthouseV2StreamUrl(sessionId),
sessionId,
submissionVersion,
);
const result = await sendLighthouseV2Message({
sessionId,
@@ -366,8 +421,23 @@ export function createLighthouseChatStore(
}));
notifyLighthouseV2SessionsChanged();
} finally {
const terminalEvent =
pendingTerminalEvent?.submissionVersion === submissionVersion
? pendingTerminalEvent
: null;
if (terminalEvent) pendingTerminalEvent = null;
if (submissionVersion === submissionIntentVersion) {
set({ isSubmitting: false });
const streamStatus = get().streamState.status;
const shouldRefreshTerminalEvent =
streamStatus === LIGHTHOUSE_V2_STREAM_STATUS.ERROR ||
streamStatus === LIGHTHOUSE_V2_STREAM_STATUS.COMPLETED;
if (terminalEvent && shouldRefreshTerminalEvent) {
await refreshAfterTerminalEvent(
terminalEvent.sessionId,
terminalEvent.event,
);
}
}
}
};
@@ -383,6 +453,7 @@ export function createLighthouseChatStore(
isSubmitting: false,
isLoadingSession: false,
lastSubmission: null,
failedOutcomeMessageId: null,
selectedModelSelection: resolveInitialModelSelection(
connectedConfigurations,
config.modelsByProvider,
@@ -395,13 +466,18 @@ export function createLighthouseChatStore(
setInput: (value) => set({ input: value }),
dismissFeedback: () => set({ feedback: null }),
dismissFeedback: () =>
set({ feedback: null, failedOutcomeMessageId: null }),
selectModel: async (selection) => {
// The selection drives the model used for the next message, so it stays
// applied even if persisting it as the provider's default model fails —
// reverting it would make a connected provider unusable when the save 4xxs.
set({ selectedModelSelection: selection, feedback: null });
set({
selectedModelSelection: selection,
feedback: null,
failedOutcomeMessageId: null,
});
const configId = connectedConfigurations.find(
(configuration) =>
@@ -449,6 +525,7 @@ export function createLighthouseChatStore(
isSubmitting: false,
isLoadingSession: true,
lastSubmission: null,
failedOutcomeMessageId: null,
streamState: createInitialLighthouseV2StreamState(),
});
syncSessionUrl(sessionId);
@@ -476,6 +553,7 @@ export function createLighthouseChatStore(
isSubmitting: false,
isLoadingSession: false,
lastSubmission: null,
failedOutcomeMessageId: null,
streamState: createInitialLighthouseV2StreamState(),
});
syncSessionUrl(null);
ui/changelog.d/lighthouse-request-outcome-feedback.added.md
+1
View File
@@ -0,0 +1 @@
Thumbs-up and thumbs-down feedback form for Lighthouse assistant answers with optional details
ui/components/survey/feedback-form.tsx
+128
View File
@@ -0,0 +1,128 @@
"use client";
import { Button } from "@/components/shadcn/button/button";
import { Textarea } from "@/components/shadcn/textarea/textarea";
interface FeedbackFormReasonOption<TReason extends string> {
value: TReason;
label: string;
}
interface FeedbackFormReasons<TReason extends string> {
label: string;
options: readonly FeedbackFormReasonOption<TReason>[];
selected: readonly TReason[];
onToggle: (reason: TReason) => void;
}
interface FeedbackFormProps<TReason extends string = string> {
title: string;
description?: string;
reasons?: FeedbackFormReasons<TReason>;
detailsLabel: string;
placeholder?: string;
details: string;
detailsMaxLength?: number;
submitLabel?: string;
isSubmitting?: boolean;
submitDisabled?: boolean;
error?: string | null;
onDetailsChange: (details: string) => void;
onSubmit: () => void;
onCancel?: () => void;
}
export function FeedbackForm<TReason extends string = string>({
title,
description,
reasons,
detailsLabel,
placeholder,
details,
detailsMaxLength,
submitLabel = "Submit",
isSubmitting = false,
submitDisabled = false,
error,
onDetailsChange,
onSubmit,
onCancel,
}: FeedbackFormProps<TReason>) {
return (
<form
className="flex flex-col gap-4"
onSubmit={(event) => {
event.preventDefault();
onSubmit();
}}
>
<div className="flex flex-col gap-1">
<h2 className="text-text-neutral-primary text-base font-semibold">
{title}
</h2>
{description ? (
<p className="text-text-neutral-secondary text-sm">{description}</p>
) : null}
</div>
{reasons ? (
<fieldset className="flex flex-col gap-2">
<legend className="text-text-neutral-primary mb-2 text-sm font-medium">
{reasons.label}
</legend>
<div className="flex flex-wrap gap-2">
{reasons.options.map((reason) => {
const selected = reasons.selected.includes(reason.value);
return (
<Button
key={reason.value}
type="button"
variant={selected ? "secondary" : "outline"}
size="xs"
aria-pressed={selected}
disabled={isSubmitting}
onClick={() => reasons.onToggle(reason.value)}
>
{reason.label}
</Button>
);
})}
</div>
</fieldset>
) : null}
<Textarea
aria-label={detailsLabel}
placeholder={placeholder}
value={details}
maxLength={detailsMaxLength}
disabled={isSubmitting}
onChange={(event) => onDetailsChange(event.target.value)}
textareaSize="lg"
className="min-h-32"
/>
{error ? (
<p role="alert" className="text-text-error text-sm">
{error}
</p>
) : null}
{onCancel ? (
<div className="flex justify-end gap-2">
<Button
type="button"
variant="outline"
disabled={isSubmitting}
onClick={onCancel}
>
Cancel
</Button>
<Button type="submit" disabled={isSubmitting || submitDisabled}>
{isSubmitting ? "Sending..." : error ? "Retry" : submitLabel}
</Button>
</div>
) : (
<Button type="submit" disabled={isSubmitting || submitDisabled}>
{isSubmitting ? "Sending..." : error ? "Retry" : submitLabel}
</Button>
)}
</form>
);
}
ui/components/survey/feedback-survey.test.tsx
+1
View File
@@ -315,6 +315,7 @@ describe("FeedbackSurvey", () => {
).toBeVisible();
const input = screen.getByPlaceholderText("Type your answer here");
expect(input).toBeVisible();
expect(input).not.toHaveAttribute("maxlength");
expect(screen.getByRole("button", { name: "Submit answer" })).toBeVisible();
});
ui/components/survey/runtime-feedback-survey.tsx
+13 -30
View File
@@ -12,7 +12,6 @@ import {
PopoverContent,
PopoverTrigger,
} from "@/components/shadcn/popover";
import { Textarea } from "@/components/shadcn/textarea/textarea";
import { useMediaQuery } from "@/hooks/use-media-query";
import { useMountEffect } from "@/hooks/use-mount-effect";
import { useStore } from "@/hooks/use-store";
@@ -24,6 +23,8 @@ import {
import { cn } from "@/lib/utils";
import { useSidePanelStore } from "@/store/side-panel";
import { FeedbackForm } from "./feedback-form";
const SURVEY_NAME = "Prowler Feedback";
const FEEDBACK_GUTTER_PX = 24;
@@ -160,35 +161,17 @@ export default function RuntimeFeedbackSurvey({
) : null}
</div>
) : (
<form
className="flex flex-col gap-4"
onSubmit={(event) => {
event.preventDefault();
handleSubmit();
}}
>
<div className="flex flex-col gap-1">
<h2 className="text-text-neutral-primary text-base font-semibold">
{question.question}
</h2>
{question.description ? (
<p className="text-text-neutral-secondary text-sm">
{question.description}
</p>
) : null}
</div>
<Textarea
aria-label={question.question}
placeholder={appearance?.placeholder ?? ""}
value={response}
onChange={(event) => setResponse(event.target.value)}
textareaSize="lg"
className="min-h-32"
/>
<Button type="submit" disabled={!trimmedResponse}>
{appearance?.submitButtonText ?? "Submit"}
</Button>
</form>
<FeedbackForm
title={question.question}
description={question.description ?? undefined}
detailsLabel={question.question}
placeholder={appearance?.placeholder ?? ""}
details={response}
submitLabel={appearance?.submitButtonText ?? "Submit"}
submitDisabled={!trimmedResponse}
onDetailsChange={setResponse}
onSubmit={handleSubmit}
/>
)}
</PopoverContent>
</Popover>