diff --git a/webui/react/src/e2e/fixtures/api.project.fixture.ts b/webui/react/src/e2e/fixtures/api.project.fixture.ts index 50d63f7213e..6a4bade3e4d 100644 --- a/webui/react/src/e2e/fixtures/api.project.fixture.ts +++ b/webui/react/src/e2e/fixtures/api.project.fixture.ts @@ -6,14 +6,9 @@ import { expect } from 'e2e/fixtures/global-fixtures'; import { safeName } from 'e2e/utils/naming'; import { ProjectsApi, V1PostProjectRequest, V1PostProjectResponse } from 'services/api-ts-sdk/api'; -import { ApiAuthFixture } from './api.auth.fixture'; - -export class ApiProjectFixture { - readonly apiAuth: ApiAuthFixture; - constructor(apiAuth: ApiAuthFixture) { - this.apiAuth = apiAuth; - } +import { apiFixture } from './api'; +export class ApiProjectFixture extends apiFixture(ProjectsApi) { new({ projectProps = {}, projectPrefix = 'test-project' } = {}): V1PostProjectRequest { const defaults = { name: safeName(projectPrefix), @@ -25,21 +20,6 @@ export class ApiProjectFixture { }; } - private static normalizeUrl(url: string): string { - if (url.endsWith('/')) { - return url.substring(0, url.length - 1); - } - return url; - } - - private async startProjectRequest(): Promise { - return new ProjectsApi( - { apiKey: await this.apiAuth.getBearerToken() }, - ApiProjectFixture.normalizeUrl(this.apiAuth.baseURL), - fetch, - ); - } - /** * Creates a project with the given parameters via the API. * @param {number} workspaceId workspace id to create the project in. @@ -53,7 +33,7 @@ export class ApiProjectFixture { workspaceId: number, req: V1PostProjectRequest, ): Promise { - const projectResp = await (await this.startProjectRequest()) + const projectResp = await this.api .postProject(workspaceId, req, {}) .catch(async function (error) { const respBody = await streamConsumers.text(error.body); @@ -75,19 +55,17 @@ export class ApiProjectFixture { await expect .poll( async () => { - const projectResp = await (await this.startProjectRequest()) - .deleteProject(id) - .catch(async function (error) { - const respBody = await streamConsumers.text(error.body); - if (error.status === 404) { - return { completed: true }; - } - throw new Error( - `Delete Project Request failed. Status: ${error.status} Request: ${JSON.stringify( - id, - )} Response: ${respBody}`, - ); - }); + const projectResp = await this.api.deleteProject(id).catch(async function (error) { + const respBody = await streamConsumers.text(error.body); + if (error.status === 404) { + return { completed: true }; + } + throw new Error( + `Delete Project Request failed. Status: ${error.status} Request: ${JSON.stringify( + id, + )} Response: ${respBody}`, + ); + }); return projectResp.completed; }, { diff --git a/webui/react/src/e2e/fixtures/api.search.fixture.ts b/webui/react/src/e2e/fixtures/api.search.fixture.ts new file mode 100644 index 00000000000..7572363fd15 --- /dev/null +++ b/webui/react/src/e2e/fixtures/api.search.fixture.ts @@ -0,0 +1,239 @@ +import { v4 as uuidV4 } from 'uuid'; + +import { safeName } from 'e2e/utils/naming'; +import { + ExperimentsApi, + InternalApi, + Trialv1State, + V1ActivateExperimentResponse, + V1ArchiveExperimentResponse, + V1CancelExperimentResponse, + V1CheckpointTrainingMetadata, + V1CreateExperimentRequest, + V1CreateExperimentResponse, + V1CreateTrialResponse, + V1DeleteExperimentResponse, + V1KillExperimentResponse, + V1PatchExperiment, + V1PatchExperimentResponse, + V1PatchTrialResponse, + V1PauseExperimentResponse, + V1PostTaskLogsResponse, + V1ReportCheckpointResponse, + V1ReportTrialMetricsResponse, + V1TaskLog, + V1TrialMetrics, + V1UnarchiveExperimentResponse, +} from 'services/api-ts-sdk'; + +import { ApiArgsFixture, apiFixture } from './api'; + +const reportApiErrorJson = >(p: T): T => { + p.catch(async (e) => { + if (e instanceof Response) console.error(await e.json()); + }); + return p; +}; + +export class ApiRun { + constructor( + protected internalApi: InternalApi, + public response: V1CreateTrialResponse, + ) {} + + get id(): number { + return this.response.trial.id; + } + + get taskId(): string { + const { taskId } = this.response.trial; + if (typeof taskId !== 'string') { + throw new Error('no task id found'); + } + return taskId; + } + + patchState(state: Trialv1State): Promise { + return reportApiErrorJson(this.internalApi.patchTrial(this.id, { state, trialId: this.id })); + } + + recordLog(logs: Omit[]): Promise { + return reportApiErrorJson( + this.internalApi.postTaskLogs({ logs: logs.map((l) => ({ ...l, taskId: this.taskId })) }), + ); + } + + reportMetrics( + group: 'training' | 'validation' | 'inference', + metrics: Omit, + ): Promise { + return reportApiErrorJson( + this.internalApi.reportTrialMetrics(this.id, { + group, + metrics: { ...metrics, trialId: this.id, trialRunId: 0 }, + }), + ); + } + + reportCheckpoint( + stepsCompleted: number, + training: V1CheckpointTrainingMetadata, + resources: { [k: string]: string } = {}, + metadata: object = {}, + ): Promise { + return reportApiErrorJson( + this.internalApi.reportCheckpoint({ + metadata: { + ...metadata, + steps_completed: stepsCompleted, + }, + resources, + state: 'STATE_COMPLETED', + taskId: this.taskId, + training, + uuid: uuidV4(), + }), + ); + } +} + +class ApiSearch { + runs: ApiRun[] = []; + + constructor( + protected experimentApi: ExperimentsApi, + protected internalApi: InternalApi, + public response: V1CreateExperimentResponse, + ) {} + + get id(): number { + return this.response.experiment.id; + } + + get externalId(): string { + const { externalExperimentId } = this.response.experiment; + if (typeof externalExperimentId !== 'string') { + throw new Error('no external experiment id found'); + } + return externalExperimentId; + } + + patch(body: Omit = {}): Promise { + return reportApiErrorJson( + this.experimentApi.patchExperiment(this.id, { id: this.id, ...body }), + ); + } + + /** + * Perform an action on a search. NOTE: unmanaged experiments can only be archived, unarchived and deleted + */ + action(action: 'pause'): Promise; + action(action: 'resume' | 'activate'): Promise; + action(action: 'cancel' | 'stop'): Promise; + action(action: 'kill'): Promise; + action(action: 'archive'): Promise; + action(action: 'unarchive'): Promise; + action(action: 'delete'): Promise; + action( + action: + | 'activate' + | 'archive' + | 'unarchive' + | 'delete' + | 'kill' + | 'pause' + | 'resume' + | 'stop' + | 'cancel', + ): Promise { + type idMethod = { + [k in keyof ExperimentsApi]: ExperimentsApi[k] extends (id: number) => Promise + ? k + : never; + }[keyof ExperimentsApi]; + const managedMethods = ['activate', 'cancel', 'kill', 'pause', 'resume', 'stop']; + const methodMap: Record = { + activate: 'activateExperiment', + archive: 'archiveExperiment', + cancel: 'cancelExperiment', + delete: 'deleteExperiment', + kill: 'killExperiment', + pause: 'pauseExperiment', + resume: 'activateExperiment', + stop: 'cancelExperiment', + unarchive: 'unarchiveExperiment', + }; + if (this.response.experiment.unmanaged && managedMethods.includes(action)) { + throw new Error(`Action ${action} only works on managed experiments`); + } + return reportApiErrorJson(this.experimentApi[methodMap[action]](this.id)); + } + + async addRun(hparams: unknown = { wow: 'cool' }): Promise { + const response = reportApiErrorJson( + this.internalApi.createTrial({ experimentId: this.id, hparams, unmanaged: true }), + ); + const apiRun = new ApiRun(this.internalApi, await response); + this.runs.push(apiRun); + return apiRun; + } + + async getRuns(): Promise { + const trials = await this.experimentApi.getExperimentTrials( + this.id, + undefined, + undefined, + undefined, + -1, + ); + this.runs = trials.trials.map((trial) => new ApiRun(this.internalApi, { trial })); + return this.runs; + } +} + +export class ApiSearchFixture extends apiFixture(InternalApi) { + searches: ApiSearch[] = []; + protected experimentApi: ExperimentsApi; + + constructor( + apiArgs: ApiArgsFixture, + public defaultProjectId: number, + ) { + super(apiArgs); + this.experimentApi = new ExperimentsApi(...apiArgs); + } + + async new( + config: object = {}, + body: Omit = {}, + ): Promise { + const configWithDefaults = { + entrypoint: 'echo bonjour!', + name: safeName('apisearch'), + searcher: { + metric: 'x', + name: 'custom', + unit: 'batches', + }, + ...config, + }; + const bodyWithDefaults = { + activate: false, + projectId: this.defaultProjectId, + ...body, + }; + const response = await reportApiErrorJson( + this.api.createExperiment({ + ...bodyWithDefaults, + config: JSON.stringify(configWithDefaults), + }), + ); + const search = new ApiSearch(this.experimentApi, this.api, response); + this.searches.push(search); + return search; + } + + dispose(): Promise { + return Promise.all(this.searches.map((s) => s.action('delete'))); + } +} diff --git a/webui/react/src/e2e/fixtures/api.ts b/webui/react/src/e2e/fixtures/api.ts new file mode 100644 index 00000000000..602d1b5fee1 --- /dev/null +++ b/webui/react/src/e2e/fixtures/api.ts @@ -0,0 +1,21 @@ +import { BaseAPI } from 'services/api-ts-sdk'; + +export type ApiArgsFixture = ConstructorParameters; +export type ApiConstructor = new (...args: ConstructorParameters) => T; + +// We need to infer the type of the resulting class here because it's a mixin +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +export const apiFixture = (api: ApiConstructor) => { + return class { + protected readonly api: T; + constructor(protected apiArgs: ApiArgsFixture) { + this.api = new api(...apiArgs); + } + protected get token() { + return this.apiArgs[0]?.apiKey; + } + protected get baseUrl() { + return this.apiArgs[1]; + } + }; +}; diff --git a/webui/react/src/e2e/fixtures/api.user.fixture.ts b/webui/react/src/e2e/fixtures/api.user.fixture.ts index 9fd34672fc3..ce8a9474201 100644 --- a/webui/react/src/e2e/fixtures/api.user.fixture.ts +++ b/webui/react/src/e2e/fixtures/api.user.fixture.ts @@ -11,14 +11,9 @@ import { V1User, } from 'services/api-ts-sdk/api'; -import { ApiAuthFixture } from './api.auth.fixture'; - -export class ApiUserFixture { - readonly apiAuth: ApiAuthFixture; - constructor(apiAuth: ApiAuthFixture) { - this.apiAuth = apiAuth; - } +import { apiFixture } from './api'; +export class ApiUserFixture extends apiFixture(UsersApi) { new({ userProps = {}, usernamePrefix = 'test-user' } = {}): V1PostUserRequest { const defaults = { isHashed: false, @@ -35,21 +30,6 @@ export class ApiUserFixture { }; } - private static normalizeUrl(url: string): string { - if (url.endsWith('/')) { - return url.substring(0, url.length - 1); - } - return url; - } - - private async startUserRequest(): Promise { - return new UsersApi( - { apiKey: await this.apiAuth.getBearerToken() }, - ApiUserFixture.normalizeUrl(this.apiAuth.baseURL), - fetch, - ); - } - /** * Creates a user with the given parameters via the API. * @param {V1PostUserRequest} req the user request with the config for the new user. @@ -59,16 +39,14 @@ export class ApiUserFixture { * strict superset of the Response, so no info is lost. */ async createUser(req: V1PostUserRequest): Promise { - const userResp = await (await this.startUserRequest()) - .postUser(req, {}) - .catch(async function (error) { - const respBody = await streamConsumers.text(error.body); - throw new Error( - `Create User Request failed. Status: ${error.status} Request: ${JSON.stringify( - req, - )} Response: ${respBody}`, - ); - }); + const userResp = await this.api.postUser(req, {}).catch(async function (error) { + const respBody = await streamConsumers.text(error.body); + throw new Error( + `Create User Request failed. Status: ${error.status} Request: ${JSON.stringify( + req, + )} Response: ${respBody}`, + ); + }); return _.merge(req, userResp); } @@ -81,16 +59,14 @@ export class ApiUserFixture { * does not include some fields like password. */ async patchUser(id: number, user: V1PatchUser): Promise { - const userResp = await (await this.startUserRequest()) - .patchUser(id, user) - .catch(async function (error) { - const respBody = await streamConsumers.text(error.body); - throw new Error( - `Patch User Request failed. Status: ${error.status} Request: ${JSON.stringify( - user, - )} Response: ${respBody}`, - ); - }); + const userResp = await this.api.patchUser(id, user).catch(async function (error) { + const respBody = await streamConsumers.text(error.body); + throw new Error( + `Patch User Request failed. Status: ${error.status} Request: ${JSON.stringify( + user, + )} Response: ${respBody}`, + ); + }); return userResp.user; } } diff --git a/webui/react/src/e2e/fixtures/api.workspace.fixture.ts b/webui/react/src/e2e/fixtures/api.workspace.fixture.ts index aaeac7558f7..82a3e383eac 100644 --- a/webui/react/src/e2e/fixtures/api.workspace.fixture.ts +++ b/webui/react/src/e2e/fixtures/api.workspace.fixture.ts @@ -10,14 +10,9 @@ import { WorkspacesApi, } from 'services/api-ts-sdk/api'; -import { ApiAuthFixture } from './api.auth.fixture'; - -export class ApiWorkspaceFixture { - readonly apiAuth: ApiAuthFixture; - constructor(apiAuth: ApiAuthFixture) { - this.apiAuth = apiAuth; - } +import { apiFixture } from './api'; +export class ApiWorkspaceFixture extends apiFixture(WorkspacesApi) { new({ workspaceProps = {}, workspacePrefix = 'test-workspace' } = {}): V1PostWorkspaceRequest { const defaults = { name: safeName(workspacePrefix), @@ -28,21 +23,6 @@ export class ApiWorkspaceFixture { }; } - private static normalizeUrl(url: string): string { - if (url.endsWith('/')) { - return url.substring(0, url.length - 1); - } - return url; - } - - private async startWorkspaceRequest(): Promise { - return new WorkspacesApi( - { apiKey: await this.apiAuth.getBearerToken() }, - ApiWorkspaceFixture.normalizeUrl(this.apiAuth.baseURL), - fetch, - ); - } - /** * Creates a workspace with the given parameters via the API. * @param {V1PostWorkspaceRequest} req the workspace request with the config for the new workspace. @@ -52,25 +32,21 @@ export class ApiWorkspaceFixture { * strict superset of the Response, so no info is lost. */ async createWorkspace(req: V1PostWorkspaceRequest): Promise { - const apiAuth = this.apiAuth; - const workspaceResp = await (await this.startWorkspaceRequest()) - .postWorkspace(req, {}) - .catch(async function (error) { - const respBody = await streamConsumers.text(error.body); - if (error.status === 401) { - const token = apiAuth.getBearerToken(); - throw new Error( - `Create Workspace Request failed. Status: ${error.status} Request: ${JSON.stringify( - req, - )} Token: ${token} Response: ${respBody}`, - ); - } + const workspaceResp = await this.api.postWorkspace(req, {}).catch(async (error) => { + const respBody = await streamConsumers.text(error.body); + if (error.status === 401) { throw new Error( `Create Workspace Request failed. Status: ${error.status} Request: ${JSON.stringify( req, - )} Response: ${respBody}`, + )} Token: ${this.token} Response: ${respBody}`, ); - }); + } + throw new Error( + `Create Workspace Request failed. Status: ${error.status} Request: ${JSON.stringify( + req, + )} Response: ${respBody}`, + ); + }); return _.merge(req, workspaceResp); } @@ -83,19 +59,17 @@ export class ApiWorkspaceFixture { await expect .poll( async () => { - const workspaceResp = await (await this.startWorkspaceRequest()) - .deleteWorkspace(id) - .catch(async function (error) { - const respBody = await streamConsumers.text(error.body); - if (error.status === 404) { - return { completed: true }; - } - throw new Error( - `Delete Workspace Request failed. Status: ${error.status} Request: ${JSON.stringify( - id, - )} Response: ${respBody}`, - ); - }); + const workspaceResp = await this.api.deleteWorkspace(id).catch(async function (error) { + const respBody = await streamConsumers.text(error.body); + if (error.status === 404) { + return { completed: true }; + } + throw new Error( + `Delete Workspace Request failed. Status: ${error.status} Request: ${JSON.stringify( + id, + )} Response: ${respBody}`, + ); + }); return workspaceResp.completed; }, { diff --git a/webui/react/src/e2e/fixtures/global-fixtures.ts b/webui/react/src/e2e/fixtures/global-fixtures.ts index be4cbf784a7..967769bede1 100644 --- a/webui/react/src/e2e/fixtures/global-fixtures.ts +++ b/webui/react/src/e2e/fixtures/global-fixtures.ts @@ -11,11 +11,11 @@ import { V1PostWorkspaceResponse, } from 'services/api-ts-sdk/api'; -// eslint-disable-next-line no-restricted-imports - +import { ApiArgsFixture } from './api'; import { ApiAuthFixture } from './api.auth.fixture'; import { ApiProjectFixture } from './api.project.fixture'; import { ApiRoleFixture } from './api.roles.fixture'; +import { ApiSearchFixture } from './api.search.fixture'; import { ApiUserFixture } from './api.user.fixture'; import { ApiWorkspaceFixture } from './api.workspace.fixture'; import { AuthFixture } from './auth.fixture'; @@ -31,6 +31,8 @@ type CustomFixtures = { apiWorkspace: ApiWorkspaceFixture; apiProject: ApiProjectFixture; authedPage: Page; + apiArgs: ApiArgsFixture; + apiSearches: ApiSearchFixture; }; type CustomWorkerFixtures = { @@ -44,10 +46,21 @@ type CustomWorkerFixtures = { backgroundApiWorkspace: ApiWorkspaceFixture; backgroundApiProject: ApiProjectFixture; backgroundAuthedPage: Page; + backgroundApiArgs: ApiArgsFixture; + backgroundApiSearches: ApiSearchFixture; +}; + +const makeApiArgs = async (auth: ApiAuthFixture): Promise => { + const { baseURL } = auth; + const normalizedURL = baseURL.endsWith('/') ? baseURL.slice(0, -1) : baseURL; + return [{ apiKey: await auth.getBearerToken() }, normalizedURL, fetch]; }; // https://playwright.dev/docs/test-fixtures export const test = baseTest.extend({ + apiArgs: async ({ apiAuth }, use) => { + await use(await makeApiArgs(apiAuth)); + }, // get the auth but allow yourself to log in through the api manually. apiAuth: async ({ playwright, browser, newAdmin, devSetup }, use) => { const apiAuth = new ApiAuthFixture(playwright.request, browser, apiUrl(), devSetup); @@ -71,18 +84,28 @@ export const test = baseTest.extend({ await use(apiAuth); }, - apiProject: async ({ apiAuth }, use) => { - const apiProject = new ApiProjectFixture(apiAuth); + apiProject: async ({ apiArgs }, use) => { + const apiProject = new ApiProjectFixture(apiArgs); await use(apiProject); }, - apiUser: async ({ apiAuth }, use) => { - const apiUser = new ApiUserFixture(apiAuth); + apiSearches: async ({ apiWorkspace, apiProject, apiArgs }, use) => { + // TODO: save everyone some time by having new call the create api + const workspaceProps = apiWorkspace.new(); + const workspace = await apiWorkspace.createWorkspace(workspaceProps); + const projectProps = apiProject.new({ projectProps: { workspaceId: workspace.workspace.id } }); + const project = await apiProject.createProject(projectProps.workspaceId, projectProps); + await use(new ApiSearchFixture(apiArgs, project.project.id)); + await apiWorkspace.deleteWorkspace(workspace.workspace.id); + }, + + apiUser: async ({ apiArgs }, use) => { + const apiUser = new ApiUserFixture(apiArgs); await use(apiUser); }, - apiWorkspace: async ({ apiAuth }, use) => { - const apiWorkspace = new ApiWorkspaceFixture(apiAuth); + apiWorkspace: async ({ apiArgs }, use) => { + const apiWorkspace = new ApiWorkspaceFixture(apiArgs); await use(apiWorkspace); }, @@ -97,6 +120,13 @@ export const test = baseTest.extend({ await use(apiAuth.page); }, + backgroundApiArgs: [ + async ({ backgroundApiAuth }, use) => { + await use(await makeApiArgs(backgroundApiAuth)); + }, + { scope: 'worker' }, + ], + /** * Does not require the pre-existing Playwright page and does not login so this can be called in beforeAll. * Generally use another api fixture instead if you want to call an api. If you just want a logged-in page, @@ -124,8 +154,8 @@ export const test = baseTest.extend({ ], backgroundApiProject: [ - async ({ backgroundApiAuth }, use) => { - const backgroundApiProject = new ApiProjectFixture(backgroundApiAuth); + async ({ backgroundApiArgs }, use) => { + const backgroundApiProject = new ApiProjectFixture(backgroundApiArgs); await use(backgroundApiProject); }, { scope: 'worker' }, @@ -137,6 +167,23 @@ export const test = baseTest.extend({ }, { scope: 'worker' }, ], + backgroundApiSearches: [ + async ({ backgroundApiWorkspace, backgroundApiProject, backgroundApiArgs }, use) => { + // TODO: save everyone some time by having new call the create api + const workspaceProps = backgroundApiWorkspace.new(); + const workspace = await backgroundApiWorkspace.createWorkspace(workspaceProps); + const projectProps = backgroundApiProject.new({ + projectProps: { workspaceId: workspace.workspace.id }, + }); + const project = await backgroundApiProject.createProject( + projectProps.workspaceId, + projectProps, + ); + await use(new ApiSearchFixture(backgroundApiArgs, project.project.id)); + await backgroundApiWorkspace.deleteWorkspace(workspace.workspace.id); + }, + { scope: 'worker' }, + ], /** * Allows calling the user api without a page so that it can run in beforeAll(). You will need to get a bearer * token by calling backgroundApiUser.apiAuth.loginAPI(). This will also provision a page in the background which @@ -144,15 +191,15 @@ export const test = baseTest.extend({ * then login() again, since setServerAddress logs out as a side effect. */ backgroundApiUser: [ - async ({ backgroundApiAuth }, use) => { - const backgroundApiUser = new ApiUserFixture(backgroundApiAuth); + async ({ backgroundApiArgs }, use) => { + const backgroundApiUser = new ApiUserFixture(backgroundApiArgs); await use(backgroundApiUser); }, { scope: 'worker' }, ], backgroundApiWorkspace: [ - async ({ backgroundApiAuth }, use) => { - const backgroundApiWorkspace = new ApiWorkspaceFixture(backgroundApiAuth); + async ({ backgroundApiArgs }, use) => { + const backgroundApiWorkspace = new ApiWorkspaceFixture(backgroundApiArgs); await use(backgroundApiWorkspace); }, { scope: 'worker' },