diff --git a/apps/web/app/(app)/automation/ReportMistake.tsx b/apps/web/app/(app)/automation/ReportMistake.tsx index dc51448e..3081cfbe 100644 --- a/apps/web/app/(app)/automation/ReportMistake.tsx +++ b/apps/web/app/(app)/automation/ReportMistake.tsx @@ -284,10 +284,7 @@ function ImproveRules({ onClick={async () => { setChecking(true); - const result = await testAiAction({ - messageId: message.id, - threadId: message.threadId, - }); + const result = await testAiAction({ messageId: message.id }); if (isActionError(result)) { toastError({ title: "There was an error testing the email", diff --git a/apps/web/app/(app)/automation/TestRules.tsx b/apps/web/app/(app)/automation/TestRules.tsx index a17ab995..71bc07dd 100644 --- a/apps/web/app/(app)/automation/TestRules.tsx +++ b/apps/web/app/(app)/automation/TestRules.tsx @@ -384,7 +384,7 @@ export function TestResultDisplay({ {isAIRule(result.rule) && (
- Rule Instructions: + AI Instructions: {result.rule.instructions.substring(0, MAX_LENGTH)} {result.rule.instructions.length >= MAX_LENGTH && "..."}
diff --git a/apps/web/package.json b/apps/web/package.json index e79b5fb7..bf8a7ae7 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -143,6 +143,7 @@ "@types/react": "18.3.12", "@types/react-dom": "18.3.1", "autoprefixer": "10.4.20", + "cross-env": "^7.0.3", "dotenv": "^16.4.7", "jiti": "^2.4.1", "jsdom": "^25.0.1", diff --git a/apps/web/utils/ai/choose-rule/ai-choose-rule.ts b/apps/web/utils/ai/choose-rule/ai-choose-rule.ts index 6771b9b9..c79733f0 100644 --- a/apps/web/utils/ai/choose-rule/ai-choose-rule.ts +++ b/apps/web/utils/ai/choose-rule/ai-choose-rule.ts @@ -46,11 +46,11 @@ ${ } `; - const prompt = `This email was received for processing. Select a rule to apply to it. + const prompt = `An email was received for processing. Select a rule to apply to it. Respond with a JSON object with the following fields: -"reason" - the reason you chose that rule. Keep it short. +"reason" - the reason you chose that rule. Keep it concise. "rule" - the number of the rule you want to apply diff --git a/apps/web/utils/ai/choose-rule/stringify-email.ts b/apps/web/utils/ai/choose-rule/stringify-email.ts index fe4d556d..490eceff 100644 --- a/apps/web/utils/ai/choose-rule/stringify-email.ts +++ b/apps/web/utils/ai/choose-rule/stringify-email.ts @@ -1,4 +1,4 @@ -import { truncate } from "@/utils/string"; +import { removeExcessiveWhitespace, truncate } from "@/utils/string"; export type EmailForLLM = { from: string; @@ -14,7 +14,7 @@ export function stringifyEmail(email: EmailForLLM, maxLength: number) { email.replyTo && `${email.replyTo}`, email.cc && `${email.cc}`, `${email.subject}`, - `${truncate(email.content, maxLength)}`, + `${truncate(removeExcessiveWhitespace(email.content), maxLength)}`, ]; return emailParts.filter(Boolean).join("\n"); diff --git a/apps/web/utils/string.test.ts b/apps/web/utils/string.test.ts new file mode 100644 index 00000000..ff2e9e8c --- /dev/null +++ b/apps/web/utils/string.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect } from "vitest"; +import { removeExcessiveWhitespace, truncate } from "./string"; + +// Run with: +// pnpm test utils/string.test.ts + +describe("string utils", () => { + describe("truncate", () => { + it("should truncate strings longer than specified length", () => { + expect(truncate("hello world", 5)).toBe("hello..."); + }); + + it("should not truncate strings shorter than specified length", () => { + expect(truncate("hello", 10)).toBe("hello"); + }); + }); + + describe("removeExcessiveWhitespace", () => { + it("should collapse multiple spaces into single space", () => { + expect(removeExcessiveWhitespace("hello world")).toBe("hello world"); + }); + + it("should preserve single newlines", () => { + expect(removeExcessiveWhitespace("hello\nworld")).toBe("hello\nworld"); + }); + + it("should collapse multiple newlines into double newlines", () => { + expect(removeExcessiveWhitespace("hello\n\n\n\nworld")).toBe( + "hello\n\nworld", + ); + }); + + it("should remove zero-width spaces", () => { + expect(removeExcessiveWhitespace("hello\u200Bworld")).toBe("hello world"); + }); + + it("should handle complex cases with multiple types of whitespace", () => { + const input = "hello world\n\n\n\n next line\u200B\u200B test"; + expect(removeExcessiveWhitespace(input)).toBe( + "hello world\n\nnext line test", + ); + }); + + it("should trim leading and trailing whitespace", () => { + expect(removeExcessiveWhitespace(" hello world ")).toBe("hello world"); + }); + + it("should handle empty string", () => { + expect(removeExcessiveWhitespace("")).toBe(""); + }); + + it("should handle string with only whitespace", () => { + expect(removeExcessiveWhitespace(" \n\n \u200B ")).toBe(""); + }); + + it("should handle soft hyphens and other special characters", () => { + const input = "hello\u00ADworld\u034Ftest\u200B\u200Cspace"; + expect(removeExcessiveWhitespace(input)).toBe("hello world test space"); + }); + + it("should handle mixed special characters and whitespace", () => { + const input = "hello\u00AD world\n\n\u034F\n\u200B test"; + expect(removeExcessiveWhitespace(input)).toBe("hello world\n\ntest"); + }); + }); +}); diff --git a/apps/web/utils/string.ts b/apps/web/utils/string.ts index 612c011b..fcb7054c 100644 --- a/apps/web/utils/string.ts +++ b/apps/web/utils/string.ts @@ -2,6 +2,28 @@ export function truncate(str: string, length: number) { return str.length > length ? `${str.slice(0, length)}...` : str; } +export function removeExcessiveWhitespace(str: string) { + return ( + str + // First remove all zero-width spaces, soft hyphens, and other invisible characters + // Handle each special character separately to avoid combining character issues + .replace( + /\u200B|\u200C|\u200D|\u200E|\u200F|\uFEFF|\u3164|\u00AD|\u034F/g, + " ", + ) + // Normalize all types of line breaks to \n + .replace(/\r\n|\r/g, "\n") + // Then collapse multiple newlines (3 or more) into double newlines + .replace(/\n\s*\n\s*\n+/g, "\n\n") + // Clean up spaces around newlines (but preserve double newlines) + .replace(/[^\S\n]*\n[^\S\n]*/g, "\n") + // Replace multiple spaces (but not newlines) with single space + .replace(/[^\S\n]+/g, " ") + // Clean up any trailing/leading whitespace + .trim() + ); +} + export function generalizeSubject(subject = "") { // replace numbers to make subject more generic // also removes [], () ,and words that start with # diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 30cbaa31..36dd1149 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -474,6 +474,9 @@ importers: autoprefixer: specifier: 10.4.20 version: 10.4.20(postcss@8.4.49) + cross-env: + specifier: ^7.0.3 + version: 7.0.3 dotenv: specifier: ^16.4.7 version: 16.4.7 @@ -6218,6 +6221,11 @@ packages: crisp-sdk-web@1.0.25: resolution: {integrity: sha512-CWTHFFeHRV0oqiXoPh/aIAKhFs6xcIM4NenGPnClAMCZUDQgQsF1OWmZWmnVNjJriXUmWRgDfeUxcxygS0dCRA==} + cross-env@7.0.3: + resolution: {integrity: sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==} + engines: {node: '>=10.14', npm: '>=6', yarn: '>=1'} + hasBin: true + cross-spawn@7.0.3: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} @@ -18434,6 +18442,10 @@ snapshots: crisp-sdk-web@1.0.25: {} + cross-env@7.0.3: + dependencies: + cross-spawn: 7.0.6 + cross-spawn@7.0.3: dependencies: path-key: 3.1.1