From 8067dbf7972bfa6617e485036969539a70cf949f Mon Sep 17 00:00:00 2001 From: texuf Date: Mon, 20 May 2024 23:05:32 -0700 Subject: [PATCH] Add better summaries to the stress tests (#23) count up the number of joins each client receives for each channel. this should let us know if they failed to sync any encryption keys --- core/stress/scripts/localhost_chat.sh | 1 + core/stress/src/mode/chat/chitChat.ts | 65 +-------------------- core/stress/src/mode/chat/joinChat.ts | 8 ++- core/stress/src/mode/chat/kickoffChat.ts | 24 ++++---- core/stress/src/mode/chat/sumarizeChat.ts | 60 +++++++++++++------ core/stress/src/start.ts | 9 +-- core/stress/src/utils/messages.ts | 71 +++++++++++++++++++++++ 7 files changed, 135 insertions(+), 103 deletions(-) create mode 100644 core/stress/src/utils/messages.ts diff --git a/core/stress/scripts/localhost_chat.sh b/core/stress/scripts/localhost_chat.sh index 9dfc2d85b..6c7fe32b3 100755 --- a/core/stress/scripts/localhost_chat.sh +++ b/core/stress/scripts/localhost_chat.sh @@ -19,6 +19,7 @@ echo "stress/scripts/localhost_chat.sh" export SPACE_ID="${SPACE_ID}" export CHANNEL_IDS="${CHANNEL_IDS}" +export ANNOUNCE_CHANNEL_ID="${ANNOUNCE_CHANNEL_ID:-}" export RIVER_ENV="${RIVER_ENV:-local_single}" export STRESS_MODE="${STRESS_MODE:-chat}" diff --git a/core/stress/src/mode/chat/chitChat.ts b/core/stress/src/mode/chat/chitChat.ts index 00b29f4a6..5b61b6f2f 100644 --- a/core/stress/src/mode/chat/chitChat.ts +++ b/core/stress/src/mode/chat/chitChat.ts @@ -1,6 +1,7 @@ import { dlogger } from '@river-build/dlog' import { StressClient } from '../../utils/stressClient' import { ChatConfig } from './types' +import { makeSillyMessage } from '../../utils/messages' export async function chitChat(client: StressClient, cfg: ChatConfig) { const logger = dlogger(`stress:chitchat:${client.logId}`) @@ -16,67 +17,3 @@ export async function chitChat(client: StressClient, cfg: ChatConfig) { await new Promise((resolve) => setTimeout(resolve, Math.random() * averateWaitTime)) } } -function makeSillyMessage() { - const w0 = [ - 'b', - 'd', - 'f', - 'g', - 'h', - 'j', - 'k', - 'l', - 'm', - 'n', - 'p', - 'r', - 's', - 't', - 'v', - 'w', - 'y', - 'z', - 'ch', - 'sh', - 'th', - 'zh', - 'ng', - ] - const w1 = ['a', 'e', 'i', 'o', 'u', 'ə', 'ɑ', 'æ', 'ɛ', 'ɪ', 'i', 'ɔ', 'ʊ', 'u', 'ʌ'] - const w2 = [ - 'ai', - 'au', - 'aw', - 'ay', - 'ea', - 'ee', - 'ei', - 'eu', - 'ew', - 'ey', - 'ie', - 'oa', - 'oi', - 'oo', - 'ou', - 'ow', - 'oy', - 'ar', - 'er', - 'ir', - 'or', - 'ur', - ] - - const wo = () => - w0[Math.floor(Math.random() * w0.length)] + w1[Math.floor(Math.random() * w1.length)] - // range over 0...Math.floor(Math.random()*3) and add a random wo - - const word = () => { - const prefix = Array.from({ length: Math.floor(Math.random() * 3) + 1 }, wo).join('') - const suffix = Math.random() > 0.8 ? w2[Math.floor(Math.random() * w2.length)] : '' - return prefix + suffix - } - - return Array.from({ length: Math.floor(Math.random() * 7) + 1 }, word).join(' ') -} diff --git a/core/stress/src/mode/chat/joinChat.ts b/core/stress/src/mode/chat/joinChat.ts index 26ad76e9a..d7fa97c9b 100644 --- a/core/stress/src/mode/chat/joinChat.ts +++ b/core/stress/src/mode/chat/joinChat.ts @@ -4,6 +4,7 @@ import { dlogger } from '@river-build/dlog' import { getRandomEmoji } from '../../utils/emoji' import { getSystemInfo } from '../../utils/systemInfo' import { channelMessagePostWhere } from '../../utils/timeline' +import { makeSillyMessage } from '../../utils/messages' export async function joinChat(client: StressClient, cfg: ChatConfig) { const logger = dlogger(`stress:joinChat:${client.logId}}`) @@ -71,10 +72,11 @@ export async function joinChat(client: StressClient, cfg: ChatConfig) { ) { await client.streamsClient.joinStream(channelId) await client.streamsClient.waitForStream(channelId) - await client.sendMessage(channelId, `I'm in! ${cfg.sessionId}`) - } else { - await client.sendMessage(channelId, `Let's go! ${cfg.sessionId}`) } + await client.sendMessage( + channelId, + `${makeSillyMessage({ maxWords: 2 })}! ${cfg.sessionId}`, + ) } logger.log('done') diff --git a/core/stress/src/mode/chat/kickoffChat.ts b/core/stress/src/mode/chat/kickoffChat.ts index 5566530fa..37a43dafd 100644 --- a/core/stress/src/mode/chat/kickoffChat.ts +++ b/core/stress/src/mode/chat/kickoffChat.ts @@ -3,6 +3,7 @@ import { getSystemInfo } from '../../utils/systemInfo' import { BigNumber } from 'ethers' import { ChatConfig } from './types' import { check, dlogger } from '@river-build/dlog' +import { makeCodeBlock } from '../../utils/messages' const logger = dlogger('stress:kickoffChat') @@ -30,24 +31,19 @@ export async function kickoffChat(rootClient: StressClient, cfg: ChatConfig) { `hello, we're starting the stress test now!, containers: ${cfg.containerCount} ppc: ${cfg.processesPerContainer} clients: ${cfg.clientsCount} sessionId: ${sessionId}`, ) - const initialStats = JSON.stringify( - { - timeToShareKeys: shareKeysDuration + 'ms', - walletBalance: balance.toString(), - testDuration: cfg.duration, - clientsCount: cfg.clientsCount, - }, - null, - 2, - ) - - const systemInfo = JSON.stringify(getSystemInfo(), null, 2) - const ticks = '```' + const initialStats = { + timeToShareKeys: shareKeysDuration + 'ms', + walletBalance: balance.toString(), + testDuration: cfg.duration, + clientsCount: cfg.clientsCount, + } logger.log('start thread') await rootClient.sendMessage( announceChannelId, - `System Info:
\n ${ticks}\n ${systemInfo}\n ${ticks}\n Initial Stats:
\n ${ticks}\n ${initialStats}\n ${ticks}\n`, + `System Info: ${makeCodeBlock(getSystemInfo())} Initial Stats: ${makeCodeBlock( + initialStats, + )}`, { threadId: eventId }, ) diff --git a/core/stress/src/mode/chat/sumarizeChat.ts b/core/stress/src/mode/chat/sumarizeChat.ts index 1592db58d..a9b517c3a 100644 --- a/core/stress/src/mode/chat/sumarizeChat.ts +++ b/core/stress/src/mode/chat/sumarizeChat.ts @@ -1,27 +1,51 @@ -import { dlogger } from '@river-build/dlog' +import { check, dlogger, shortenHexString } from '@river-build/dlog' import { StressClient } from '../../utils/stressClient' import { ChatConfig } from './types' import { getSystemInfo } from '../../utils/systemInfo' import { channelMessagePostWhere } from '../../utils/timeline' +import { isDefined } from '@river/sdk' +import { makeCodeBlock } from '../../utils/messages' -export async function sumarizeChat(client: StressClient, cfg: ChatConfig) { +export async function sumarizeChat(localClients: StressClient[], cfg: ChatConfig) { const logger = dlogger('stress:sumarizeChat') - - logger.log('sumarizeChat', client.connection.userId) - const announceChannelId = cfg.announceChannelId - const defaultChannel = await client.streamsClient.waitForStream(announceChannelId) - // find the message in the default channel that contains the session id, emoji it - const message = await client.waitFor( - () => - defaultChannel.view.timeline.find( - channelMessagePostWhere((value) => value.body.includes(cfg.sessionId)), - ), - { interval: 1000, timeoutMs: cfg.waitForChannelDecryptionTimeoutMs }, + const processLeadClient = localClients[0] + logger.log('sumarizeChat', processLeadClient.connection.userId) + const defaultChannel = processLeadClient.streamsClient.stream(cfg.announceChannelId) + check(isDefined(defaultChannel), 'defaultChannel not found') + // find the message in the default channel that contains the session id, this should already be there decrypted + const message = defaultChannel.view.timeline.find( + channelMessagePostWhere((value) => value.body.includes(cfg.sessionId)), ) + check(isDefined(message), 'message not found') - await client.sendMessage( - announceChannelId, - `c${cfg.containerIndex}p${cfg.processIndex} Done freeMemory: ${getSystemInfo().FreeMemory}`, - { threadId: message.hashStr }, - ) + const checkinCounts: Record> = {} + + // loop over clients and do summaries + for (const client of localClients) { + for (const channelId of cfg.channelIds) { + // for each channel, count the number of joinChat checkins we got (look for sessionId) + const messages = client.streamsClient.stream(channelId)?.view.timeline + const checkInMesssages = + messages?.filter( + channelMessagePostWhere((value) => value.body.includes(cfg.sessionId)), + ) ?? [] + + const key = shortenHexString(channelId) + checkinCounts[key] = { + ...checkinCounts[key], + [checkInMesssages.length.toString()]: checkInMesssages.length, + } + } + } + + const summary = { + containerIndex: cfg.containerIndex, + processIndex: cfg.processIndex, + freeMemory: getSystemInfo().FreeMemory, + checkinCounts, + } + + await processLeadClient.sendMessage(cfg.announceChannelId, `Done ${makeCodeBlock(summary)}`, { + threadId: message.hashStr, + }) } diff --git a/core/stress/src/start.ts b/core/stress/src/start.ts index 9c2009559..096c306fe 100644 --- a/core/stress/src/start.ts +++ b/core/stress/src/start.ts @@ -51,9 +51,10 @@ function getChatConfig(): ChatConfig { const clientEndIndex = clientStartIndex + clientsPerProcess const spaceId = process.env.SPACE_ID const channelIds = process.env.CHANNEL_IDS.split(',') - const announceChannelId = process.env.ANNOUNCE_CHANNEL_ID - ? process.env.ANNOUNCE_CHANNEL_ID - : makeDefaultChannelStreamId(spaceId) + const announceChannelId = + process.env.ANNOUNCE_CHANNEL_ID && process.env.ANNOUNCE_CHANNEL_ID.length > 0 + ? process.env.ANNOUNCE_CHANNEL_ID + : makeDefaultChannelStreamId(spaceId) const allWallets = generateWalletsFromSeed(getRootWallet().mnemonic, 0, clientsCount) const wallets = allWallets.slice(clientStartIndex, clientEndIndex) if (clientStartIndex >= clientEndIndex) { @@ -140,7 +141,7 @@ async function startStressChat() { await Promise.all(clients.map((client) => chitChat(client, chatConfig))) logger.log('sumarizeChat') - await sumarizeChat(clients[0], chatConfig) + await sumarizeChat(clients, chatConfig) logger.log('done') } diff --git a/core/stress/src/utils/messages.ts b/core/stress/src/utils/messages.ts new file mode 100644 index 000000000..12eb48c82 --- /dev/null +++ b/core/stress/src/utils/messages.ts @@ -0,0 +1,71 @@ +export function makeSillyMessage(opts?: { maxWords?: number }) { + const maxWords = opts?.maxWords ?? 7 + const w0 = [ + 'b', + 'd', + 'f', + 'g', + 'h', + 'j', + 'k', + 'l', + 'm', + 'n', + 'p', + 'r', + 's', + 't', + 'v', + 'w', + 'y', + 'z', + 'ch', + 'sh', + 'th', + 'zh', + 'ng', + ] + const w1 = ['a', 'e', 'i', 'o', 'u', 'ə', 'ɑ', 'æ', 'ɛ', 'ɪ', 'i', 'ɔ', 'ʊ', 'u', 'ʌ'] + const w2 = [ + 'ai', + 'au', + 'aw', + 'ay', + 'ea', + 'ee', + 'ei', + 'eu', + 'ew', + 'ey', + 'ie', + 'oa', + 'oi', + 'oo', + 'ou', + 'ow', + 'oy', + 'ar', + 'er', + 'ir', + 'or', + 'ur', + ] + + const wo = () => + w0[Math.floor(Math.random() * w0.length)] + w1[Math.floor(Math.random() * w1.length)] + + const word = () => { + const prefix = Array.from({ length: Math.floor(Math.random() * 3) + 1 }, wo).join('') + const suffix = Math.random() > 0.8 ? w2[Math.floor(Math.random() * w2.length)] : '' + return prefix + suffix + } + + return Array.from({ length: Math.floor(Math.random() * (maxWords - 1)) + 1 }, word).join(' ') +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function makeCodeBlock(value: any): string { + const str = JSON.stringify(value, null, 2) + const ticks = '```' + return `
\n ${ticks}\n${str}\n${ticks}\n` +}