Merge branch 'dev' of https://github.com/efdevcon/monorepo into dev

devcon-api/package.json

@@ -65,6 +65,7 @@
     "@octokit/rest": "^19.0.7",
     "@prisma/client": "^5.17.0",
     "better-sqlite3": "^11.1.2",
+    "connect-pg-simple": "^10.0.0",
     "cors": "^2.8.5",
     "cross-fetch": "^3.1.5",
     "csv-parser": "^3.0.0",
@@ -81,11 +82,10 @@
     "jose": "^5.9.6",
     "llamaindex": "^0.3.10",
     "markdown-it": "^13.0.1",
-    "memorystore": "^1.6.7",
     "natural": "^7.1.0",
     "nodemailer": "^6.9.13",
     "openai": "^4.55.5",
-    "pg": "^8.12.0",
+    "pg": "^8.13.1",
     "puppeteer": "18.2.1",
     "rate-limiter-flexible": "^5.0.3",
     "remark": "^15.0.1",
@@ -104,6 +104,7 @@
     "@octokit/types": "^9.0.0",
     "@tsconfig/recommended": "^1.0.2",
     "@types/better-sqlite3": "^7.6.11",
+    "@types/connect-pg-simple": "^7.0.3",
     "@types/cors": "^2.8.13",
     "@types/dotenv": "^8.2.0",
     "@types/express": "^4.17.15",

devcon-api/src/app.ts

@@ -10,7 +10,8 @@ import { notFoundHandler } from '@/middleware/notfound'
 import { logHandler } from '@/middleware/log'
 import { router } from './routes'
 import { SERVER_CONFIG, SESSION_CONFIG } from '@/utils/config'
-import createMemoryStore from 'memorystore'
+import pgSession from 'connect-pg-simple'
+import { getDbPool } from './utils/db'
 
 const app = express()
 
@@ -27,15 +28,16 @@ app.use(
   })
 )
 
-const store = createMemoryStore(session)
+const pgSessionStore = pgSession(session)
 const sessionConfig: SessionOptions = {
   name: SESSION_CONFIG.cookieName,
   secret: SESSION_CONFIG.password,
   cookie: {},
   resave: false,
   saveUninitialized: true,
-  store: new store({
-    checkPeriod: 86400000, // prune expired entries every 24h
+  store: new pgSessionStore({
+    pool: getDbPool(),
+    tableName: 'Session',
   }),
 }
 

devcon-api/src/clients/recommendation.ts

@@ -1,15 +1,44 @@
 import { PrismaClient as ScheduleClient } from '@prisma/client'
-import { PrismaClient as AccountClient } from '@/db/clients/account'
+import { Account, PrismaClient as AccountClient } from '@/db/clients/account'
 import { SERVER_CONFIG } from '@/utils/config'
+import { Session } from '@/types/schedule'
+import { STOPWORDS } from '@/utils/stopwords'
+import { writeFileSync } from 'fs'
+import dictionary from '../../data/vectors/dictionary.json'
+import vectorizedSessions from '../../data/vectors/devcon-7.json'
+
+export const WEIGHTS = {
+  track: 6,
+  expertise: 4,
+  audience: 4,
+  speaker: 6,
+  tag: 2,
+  featured: 0.1,
+}
+
+export interface VectorizedSession {
+  session: Session
+  vector: number[]
+}
 
-interface LensFollower {
+export interface VectorDictionary {
+  tracks: string[]
+  speakers: string[]
+  tags: string[]
+  expertise: string[]
+  audiences: string[]
+}
+
+export interface LensFollower {
   id: string
   fullHandle: string
   handle: string
   address: string
   ens: string
 }
 
+let cachedDictionary: VectorDictionary = dictionary
+
 const scheduleClient = new ScheduleClient()
 const accountClient = new AccountClient()
 
@@ -58,22 +87,53 @@ export async function GetRecommendedSessions(id: string, includeFeatured?: boole
     return []
   }
 
+  const userVector = vectorizeUser(account)
+  const personalizedRecommendations = GetRecommendedVectorSearch(userVector, vectorizedSessions as VectorizedSession[], 20)
   const sessions = await scheduleClient.session.findMany({
     where: {
       AND: [
-        // { eventId: 'devcon-7' },
+        { eventId: 'devcon-7' },
         {
-          OR: [{ featured: true }, { speakers: { some: { id: { in: account.favorite_speakers } } } }],
+          OR: [
+            includeFeatured ? { featured: true } : {},
+            { speakers: { some: { id: { in: account.favorite_speakers } } } },
+            { id: { in: personalizedRecommendations.map((r) => r.id) } },
+          ],
         },
       ],
     },
   })
 
-  // Find related sessions based on account.tracks, account.tags, etc. from RelatedSessions
-
   return sessions
 }
 
+export function GetRecommendedVectorSearch(sessionVector: number[], allSessions: VectorizedSession[], limit: number = 10): Session[] {
+  const similarities = allSessions
+    .filter((vs) => vs.vector !== sessionVector)
+    .map((vs) => {
+      const vectorSimilarity = getSimilarity(sessionVector, vs.vector)
+      const featuredBoost = vs.session.featured ? WEIGHTS.featured : 0
+      const adjustedSimilarity = vectorSimilarity + featuredBoost
+
+      return {
+        session: vs.session,
+        similarity: adjustedSimilarity,
+      }
+    })
+
+  const recommendations = similarities
+    .sort((a, b) => b.similarity - a.similarity)
+    .slice(0, limit)
+    .map((item) => {
+      return {
+        ...item.session,
+        similarity: item.similarity,
+      }
+    })
+
+  return recommendations
+}
+
 export async function GetFarcasterFollowing(profileId: string, cursor?: string): Promise<any[]> {
   console.log('Get Farcaster Following', profileId, cursor)
 
@@ -261,3 +321,118 @@ export async function GetLensProfileId(id: string) {
     console.error('Error fetching social followers:', error)
   }
 }
+
+export function buildDictionary(sessions: Session[], rebuild: boolean = false) {
+  if (cachedDictionary && !rebuild) return cachedDictionary
+
+  const allTracks = Array.from(new Set(sessions.map((s) => s.track))).filter((t) => !STOPWORDS.includes(t))
+  const allSpeakers = Array.from(new Set(sessions.flatMap((s) => s.speakers))).filter((s) => !STOPWORDS.includes(s))
+  const allTags = Array.from(new Set(sessions.flatMap((s) => s.tags))).filter((t) => !STOPWORDS.includes(t))
+  const allExpertise = Array.from(new Set(sessions.map((s) => s.expertise))).filter((e) => !STOPWORDS.includes(e))
+  const allAudiences = Array.from(new Set(sessions.map((s) => s.audience))).filter((a) => !STOPWORDS.includes(a))
+
+  return { tracks: allTracks, speakers: allSpeakers, tags: allTags, expertise: allExpertise, audiences: allAudiences } as VectorDictionary
+}
+
+export function vectorizeSessions(sessions: any[], limit: number = 10, saveToFile?: boolean) {
+  const dictionary = buildDictionary(sessions, true)
+  const vectorizedSessions: VectorizedSession[] = sessions.map((session) => ({
+    session,
+    vector: vectorizeSession(session, dictionary),
+  }))
+
+  if (saveToFile) {
+    writeFileSync(`data/vectors/dictionary.json`, JSON.stringify(dictionary, null, 2))
+    writeFileSync(`data/vectors/devcon-7.json`, JSON.stringify(vectorizedSessions, null, 2))
+  }
+
+  const similarities = []
+  for (let i = 0; i < vectorizedSessions.length; i++) {
+    const session = vectorizedSessions[i]
+    const recommendations = GetRecommendedVectorSearch(session.vector, vectorizedSessions, limit)
+    similarities.push(
+      ...recommendations.map((rec) => ({
+        sessionId: session.session.id,
+        otherId: rec.id,
+        similarity: rec.similarity || 0,
+      }))
+    )
+  }
+
+  return similarities
+}
+
+export function vectorizeSession(session: Session, dictionary: VectorDictionary): number[] {
+  const vector = [
+    ...dictionary.tracks.map((track) => (session.track === track ? 1 : 0)),
+    ...dictionary.speakers.map((speaker) => (session.speakers.includes(speaker) ? 1 : 0)),
+    ...dictionary.tags.map((tag) => (session.tags.includes(tag) ? 1 : 0)),
+    ...dictionary.expertise.map((exp) => (session.expertise === exp ? 1 : 0)),
+    ...dictionary.audiences.map((aud) => (session.audience === aud ? 1 : 0)),
+  ]
+
+  return getVectorWeight(vector, dictionary)
+}
+
+export function vectorizeUser(user: Account, dic: VectorDictionary = dictionary): number[] {
+  const vector = [
+    ...dictionary.tracks.map((track) => (user.tracks.includes(track) ? 1 : 0)),
+    ...dictionary.speakers.map((speaker) => (user.favorite_speakers.includes(speaker) ? 1 : 0)),
+    ...dictionary.tags.map((tag) => (user.tags.includes(tag) ? 1 : 0)),
+    ...dictionary.expertise.map((exp) => (getExpertiseLevel(user?.since).includes(exp) ? 1 : 0)),
+    ...dictionary.audiences.map((aud) => (user.roles.includes(aud) ? 1 : 0)),
+  ]
+
+  return getVectorWeight(vector, dictionary)
+}
+
+export function getVectorWeight(vector: number[], dictionary: VectorDictionary) {
+  const trackLength = dictionary.tracks.length
+  const expertiseLength = dictionary.expertise.length
+  const audienceLength = dictionary.audiences.length
+  const speakerLength = dictionary.speakers.length
+  const tagLength = dictionary.tags.length
+
+  for (let i = 0; i < vector.length; i++) {
+    if (i < trackLength) {
+      vector[i] *= WEIGHTS.track
+    } else if (i < trackLength + expertiseLength) {
+      vector[i] *= WEIGHTS.expertise
+    } else if (i < trackLength + expertiseLength + audienceLength) {
+      vector[i] *= WEIGHTS.audience
+    } else if (i < trackLength + expertiseLength + audienceLength + speakerLength) {
+      vector[i] *= WEIGHTS.speaker
+    } else if (i < trackLength + expertiseLength + audienceLength + speakerLength + tagLength) {
+      vector[i] *= WEIGHTS.tag
+    }
+  }
+
+  return vector
+}
+
+export function getSimilarity(vector1: number[], vector2: number[]): number {
+  if (vector1.length !== vector2.length) {
+    throw new Error('Vectors must have the same length')
+  }
+
+  const dotProduct = vector1.reduce((acc, val, index) => acc + val * vector2[index], 0)
+  const magnitude1 = Math.sqrt(vector1.reduce((acc, val) => acc + Math.pow(val, 2), 0))
+  const magnitude2 = Math.sqrt(vector2.reduce((acc, val) => acc + Math.pow(val, 2), 0))
+
+  if (magnitude1 === 0 || magnitude2 === 0) {
+    return 0
+  }
+
+  return dotProduct / (magnitude1 * magnitude2)
+}
+
+export function getExpertiseLevel(since?: number | null) {
+  if (!since) return []
+
+  if (since >= 2023) return ['Beginner']
+  if (since >= 2021) return ['Beginner', 'Intermediate']
+  if (since >= 2019) return ['Intermediate']
+  if (since >= 2017) return ['Intermediate', 'Advanced']
+
+  return ['Advanced']
+}

devcon-api/src/controllers/sessions.ts

@@ -219,13 +219,13 @@ export async function GetSessionRelated(req: Request, res: Response) {
   const data = await client.relatedSession.findMany({
     where: { sessionId: req.params.id },
     orderBy: { similarity: 'desc' },
-    include: { related: true },
+    include: { other: true },
     take: 10,
   })
 
   if (!data) return res.status(404).send({ status: 404, message: 'Not Found' })
 
-  res.status(200).send({ status: 200, message: '', data: data.map((i) => i.related) })
+  res.status(200).send({ status: 200, message: '', data: data.map((i) => i.other) })
 }
 
 export async function GetSessionImage(req: Request, res: Response) {

devcon-api/src/db/account.prisma

@@ -42,6 +42,14 @@ model Account {
   updatedAt DateTime?
 }
 
+model Session {
+  sid    String   @id @db.VarChar
+  sess   Json
+  expire DateTime @db.Timestamp(6)
+
+  @@index([expire], name: "IDX_session_expire")
+}
+
 model rate_limit {
   key    String  @id @db.VarChar(255)
   points Int     @default(0)

devcon-api/src/db/migrations/20241026175112_add_session_model/migration.sql

@@ -0,0 +1,11 @@
+-- CreateTable
+CREATE TABLE "Session" (
+    "sid" VARCHAR NOT NULL,
+    "sess" JSONB NOT NULL,
+    "expire" TIMESTAMP(6) NOT NULL,
+
+    CONSTRAINT "Session_pkey" PRIMARY KEY ("sid")
+);
+
+-- CreateIndex
+CREATE INDEX "IDX_session_expire" ON "Session"("expire");

devcon-api/src/db/schema.prisma

@@ -85,14 +85,27 @@ model Session {
   resources_slides String?
 
   relatedSessions RelatedSession[] @relation(name: "Related")
+  recommendedSessions RecommendedSession[] @relation(name: "Recommended")
 }
 
 model RelatedSession {
   id Int @id @default(autoincrement())
   sessionId String
+  vector Bytes?
 
-  related Session @relation(name: "Related", fields: [relatedId], references: [id])
-  relatedId String
+  other Session @relation(name: "Related", fields: [otherId], references: [id])
+  otherId String
 
   similarity Float
 }
+
+model RecommendedSession {
+  id Int @id @default(autoincrement())
+  sessionId String
+  vector Bytes?
+
+  other Session @relation(name: "Recommended", fields: [otherId], references: [id])
+  otherId String
+  
+  similarity Float
+}