Merge pull request #4 from digidem/background-jobs

feat: add basic background jobs #9

.python-version

@@ -0,0 +1 @@
+3.11

apps/dashboard/package.json

@@ -8,7 +8,7 @@
     "lint": "biome lint",
     "format": "biome format --write .",
     "start": "next start",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc-files --noEmit"
   },
   "dependencies": {
     "@eda/analytics": "workspace:*",

apps/docs/package.json

@@ -11,7 +11,7 @@
     "clean": "git clean -xdf .next .turbo node_modules",
     "lint": "biome lint src",
     "format": "biome format --write .",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc-files --noEmit"
   },
   "dependencies": {
     "@astrojs/check": "^0.9.4",

apps/whatsapp/package.json

@@ -17,7 +17,7 @@
     "typescript": "^5.0.0"
   },
   "dependencies": {
-    "@trigger.dev/sdk": "3.1.0",
+    "@trigger.dev/sdk": "3.2.1",
     "dotenv": "^16.4.5",
     "mkdirp": "^3.0.1",
     "mongoose": "^8.7.1",

assets/DEVELOPMENT.md

@@ -96,8 +96,8 @@ cp deploy/langtrace-stack/.env.example deploy/langtrace-stack/.env
 
 3. Before starting the development server, ensure you have the required environmental variables in place:
 
-- Connect to Trigger instance by setting the correct `TRIGGER_PROJECT_ID` and `TRIGGER_API_URL` variables in the `packages/jobs/.env` file from [Local Trigger](http://localhost:3001/) or [Cloud Trigger](https://cloud.trigger.dev)
-- Add the correct `TRIGGER_SECRET_KEY` to `apps/whatsapp/.env` and `packages/simulator/.env` from [Local Trigger](http://localhost:3001/) or [Cloud Trigger](https://cloud.trigger.dev) ([docs](https://trigger.dev/docs/apikeys))
+- Connect to Trigger instance by setting the correct `TRIGGER_PROJECT_ID` and `TRIGGER_API_URL` variables in the `packages/jobs/.env` file from [Local Trigger](http://localhost:3040/) or [Cloud Trigger](https://cloud.trigger.dev)
+- Add the correct `TRIGGER_SECRET_KEY` to `apps/whatsapp/.env` and `packages/simulator/.env` from [Local Trigger](http://localhost:3040/) or [Cloud Trigger](https://cloud.trigger.dev) ([docs](https://trigger.dev/docs/apikeys))
 - Add the correct `SUPABASE_SERVICE_ROLE_KEY` to the `packages/jobs/.env` from [Local Supabase](http://localhost:54323/project/default/settings/api) or [Cloud Supabase](https://supabase.com/dashboard/)
 - Add valid `CEREBRAS_API_KEY` or `OPENAI_API_KEY` to `apps/ai_api/.env` from [Cerebras](https://cloud.cerebras.ai/platform) or [OpenAI](https://platform.openai.com/api-keys)
 - (optional) Add the correct `LANGTRACE_API_KEY` to the `.env` in `apps/ai_api/.env` and `packages/simulator/.env`
@@ -142,7 +142,7 @@ bun seed // run Supabase seed
 - **Documentation**: Visit [http://localhost:8082](http://localhost:8082) to browse the documentation.
 - **AI API**: Visit [http://localhost:8083/docs](http://localhost:8083/docs) to access the API documentation.
 - **Supabase Studio**: Visit [http://localhost:54323](http://localhost:54323) to manage your database, view API documentation, and perform backend tasks.
-- **Trigger.dev Dashboard**: Visit [http://localhost:3001](http://localhost:3001) to manage jobs. Use `docker logs -f trigger-webapp` to view the magic link in the logs.
+- **Trigger.dev Dashboard**: Visit [http://localhost:3040](http://localhost:3040) to manage jobs. Use `docker logs -f trigger-webapp` to view the magic link in the logs.
 
 Development should be done primarily through the simulator, which creates realistic scenarios of community members interacting with the system. The simulator enables testing conversations between two AI agents - one representing a community member and another representing the assistant.
 

deploy/trigger-stack/.env.example

@@ -1,4 +1,4 @@
-PORT=3001
+PORT=3040
 REMIX_APP_PORT=3030
 
 NODE_ENV=production

package.json

@@ -34,9 +34,10 @@
     "@biomejs/biome": "1.9.4",
     "@t3-oss/env-nextjs": "^0.11.1",
     "lefthook": "^1.8.2",
+    "tsc-files": "^1.1.4",
     "turbo": "^2.2.3",
     "typescript": "^5.6.3"
   },
-  "packageManager": "[email protected].33",
+  "packageManager": "[email protected].34",
   "private": true
 }

packages/analytics/package.json

@@ -8,7 +8,7 @@
     "clean": "rm -rf .turbo node_modules",
     "lint": "biome check .",
     "format": "biome --write .",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc-files --noEmit"
   },
   "dependencies": {
     "@eda/logger": "workspace:*",

packages/email/package.json

@@ -5,7 +5,7 @@
     "clean": "rm -rf .turbo node_modules",
     "lint": "biome lint ./components ./emails",
     "format": "biome lint --write --unsafe",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "tsc-files --noEmit",
     "dev": "email dev -p 8083",
     "build": "email build",
     "start": "email start"

packages/jobs/package.json

@@ -7,15 +7,17 @@
     "lint": "biome lint ./trigger ./lib",
     "dev": "bunx trigger.dev dev",
     "format": "biome format --write .",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc-files --noEmit",
+    "test": "vitest"
   },
   "dependencies": {
     "@eda/supabase": "workspace:*",
     "@eda/types": "workspace:*",
-    "@getzep/zep-cloud": "^2.0.0",
-    "@trigger.dev/sdk": "3.1.0"
+    "@getzep/zep-cloud": "^2.1.1",
+    "@trigger.dev/sdk": "3.2.1"
   },
   "devDependencies": {
-    "@trigger.dev/build": "3.1.0"
+    "@trigger.dev/build": "3.2.1",
+    "vitest": "^2.1.5"
   }
 }

packages/jobs/schemas/grant-calls.schema.ts

@@ -0,0 +1,66 @@
+import { z } from "zod";
+
+// Define specific types for meta values
+const metaValueSchema = z.union([
+  z.string(),
+  z.number(),
+  z.boolean(),
+  z.null(),
+  z.array(z.union([z.string(), z.number(), z.boolean(), z.null()])),
+  z.record(
+    z.string(),
+    z.union([z.string(), z.number(), z.boolean(), z.null()]),
+  ),
+]);
+
+export const querySchema = z
+  .object({
+    userId: z.string().optional(),
+    limit: z.number().int().min(0).optional(),
+    offset: z.number().int().min(0).optional(),
+    status: z.enum(["open", "closed", "draft"]).optional(),
+    focusArea: z.string().optional(),
+    minAmount: z.number().min(0).optional(),
+    maxAmount: z.number().min(0).optional(),
+  })
+  .refine(
+    (data) =>
+      !data.minAmount || !data.maxAmount || data.maxAmount >= data.minAmount,
+    { message: "maxAmount must be greater than or equal to minAmount" },
+  );
+
+export const saveGrantCallSchema = z.object({
+  id: z.string().optional(),
+  title: z.string(),
+  description: z.string(),
+  organization: z.string(),
+  funding_amount: z.number().min(0),
+  deadline: z.string(),
+  focus_areas: z.array(z.string()),
+  eligibility_criteria: z.array(z.string()),
+  requirements: z.array(z.string()),
+  status: z.enum(["open", "closed", "draft"]),
+  meta: z.record(z.string(), metaValueSchema).optional(),
+});
+
+export type MetaValue =
+  | string
+  | number
+  | boolean
+  | null
+  | Array<string | number | boolean | null>
+  | Record<string, string | number | boolean | null>;
+
+export type GrantCallResult = {
+  id: string;
+  title: string;
+  description: string;
+  organization: string;
+  funding_amount: number;
+  deadline: string;
+  focus_areas: string[];
+  eligibility_criteria: string[];
+  requirements: string[];
+  status: "open" | "closed" | "draft";
+  meta?: Record<string, MetaValue>;
+};

packages/jobs/schemas/profiles.schema.ts

@@ -0,0 +1,55 @@
+import { z } from "zod";
+
+// Define specific types for meta values
+const metaValueSchema = z.union([
+  z.string(),
+  z.number(),
+  z.boolean(),
+  z.null(),
+  z.array(z.union([z.string(), z.number(), z.boolean(), z.null()])),
+  z.record(
+    z.string(),
+    z.union([z.string(), z.number(), z.boolean(), z.null()]),
+  ),
+]);
+
+// Query schema for getting profiles
+export const querySchema = z.object({
+  userId: z.string().optional(),
+  biome: z.string().optional(),
+  ethnicGroup: z.string().optional(),
+  territory: z.string().optional(),
+  community: z.string().optional(),
+  limit: z.number().int().min(0).optional(),
+  offset: z.number().int().min(0).optional(),
+});
+
+// Schema for saving profiles
+export const saveProfileSchema = z.object({
+  id: z.string().optional(),
+  userId: z.string().min(1),
+  biome: z.string().min(1),
+  ethnicGroup: z.string().min(1),
+  territory: z.string().min(1),
+  community: z.string().min(1),
+  meta: z.record(z.string(), metaValueSchema).optional(),
+});
+
+export type MetaValue =
+  | string
+  | number
+  | boolean
+  | null
+  | Array<string | number | boolean | null>
+  | Record<string, string | number | boolean | null>;
+
+export type ProfileResult = {
+  id: string;
+  user_id: string;
+  biome: string;
+  ethnic_group: string;
+  territory: string;
+  community: string;
+  meta?: Record<string, MetaValue>;
+  created_at: string;
+};

packages/jobs/schemas/received-messages.schema.ts

@@ -0,0 +1,43 @@
+import { z } from "zod";
+
+export const querySchema = z
+  .object({
+    userId: z.string().optional(),
+    limit: z.number().int().min(0).optional(),
+    offset: z.number().int().min(0).optional(),
+    fromTimestamp: z.number().min(0).optional(),
+    toTimestamp: z.number().min(0).optional(),
+  })
+  .refine(
+    (data) =>
+      !data.fromTimestamp ||
+      !data.toTimestamp ||
+      data.toTimestamp >= data.fromTimestamp,
+    { message: "toTimestamp must be greater than or equal to fromTimestamp" },
+  );
+
+export const messageSchema = z.object({
+  id: z.string(),
+  _id: z.string(),
+  from: z.string(),
+  to: z.string(),
+  userId: z.string(),
+  data: z.object({
+    text: z.string(),
+  }),
+  timestamp: z.number(),
+  type: z.string(),
+  isGroupChat: z.boolean(),
+  chatId: z.string(),
+  created: z.string(),
+});
+
+export type MessageResult = {
+  id: string;
+  user_id: string;
+  text: string;
+  timestamp: number;
+  processed: boolean;
+  meta: Record<string, unknown>;
+  created_at: string;
+};

packages/jobs/schemas/tosend-messages.schema.ts

@@ -0,0 +1,47 @@
+import { z } from "zod";
+
+export const querySchema = z
+  .object({
+    userId: z.string().optional(),
+    limit: z.number().int().min(0).optional(),
+    offset: z.number().int().min(0).optional(),
+    fromTimestamp: z.number().min(0).optional(),
+    toTimestamp: z.number().min(0).optional(),
+    processed: z.boolean().optional(),
+  })
+  .refine(
+    (data) =>
+      !data.fromTimestamp ||
+      !data.toTimestamp ||
+      data.toTimestamp >= data.fromTimestamp,
+    { message: "toTimestamp must be greater than or equal to fromTimestamp" },
+  );
+
+const metaValueSchema = z.union([
+  z.string(),
+  z.number(),
+  z.boolean(),
+  z.null(),
+  z.array(z.union([z.string(), z.number(), z.boolean(), z.null()])),
+  z.record(
+    z.string(),
+    z.union([z.string(), z.number(), z.boolean(), z.null()]),
+  ),
+]);
+
+const messageSchema = z.object({
+  userId: z.string().min(1),
+  text: z.string().min(1),
+  timestamp: z.number(),
+  meta: z.record(z.string(), metaValueSchema).optional(),
+});
+
+export type MetaValue =
+  | string
+  | number
+  | boolean
+  | null
+  | Array<string | number | boolean | null>
+  | Record<string, string | number | boolean | null>;
+
+export { messageSchema };