From dc90f20e46e90d3ad48060e00b8f42b17e2e8324 Mon Sep 17 00:00:00 2001 From: Oleg Solomka Date: Thu, 21 Nov 2024 17:18:36 -0800 Subject: [PATCH] [prompt snippets]: move more codec files around --- src/vs/editor/common/codecs/baseToken.ts | 18 +++- .../common/codecs/linesCodec/tokens/line.ts | 8 +- .../codecs/linesCodec/tokens/newLine.ts | 7 -- .../common/codecs/simpleCodec/tokens/space.ts | 7 -- .../common/codecs/simpleCodec/tokens/tab.ts | 7 -- .../common/codecs/simpleCodec/tokens/word.ts | 8 +- .../editor/test/common/utils/testDecoder.ts | 92 +------------------ .../contrib/chat/browser/chatInputPart.ts | 4 +- .../chatbotPromptCodec/chatbotPromptCodec.ts | 8 +- .../chatbotPromptDecoder.ts | 10 +- .../tokens/fileReference.ts | 15 ++- .../chat/common/promptFileReference.ts | 2 +- .../common/codecs/chatbotPromptCodec.test.ts | 10 +- .../codecs/chatbotPromptDecoder.test.ts | 10 +- 14 files changed, 66 insertions(+), 140 deletions(-) rename src/vs/{editor => workbench/contrib/chat}/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts (76%) rename src/vs/{editor => workbench/contrib/chat}/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts (74%) rename src/vs/{editor => workbench/contrib/chat}/common/codecs/chatbotPromptCodec/tokens/fileReference.ts (83%) rename src/vs/{editor => workbench/contrib/chat}/test/common/codecs/chatbotPromptCodec.test.ts (87%) rename src/vs/{editor => workbench/contrib/chat}/test/common/codecs/chatbotPromptDecoder.test.ts (86%) diff --git a/src/vs/editor/common/codecs/baseToken.ts b/src/vs/editor/common/codecs/baseToken.ts index e2b8a3333ed69..b25fa2184899f 100644 --- a/src/vs/editor/common/codecs/baseToken.ts +++ b/src/vs/editor/common/codecs/baseToken.ts @@ -9,7 +9,7 @@ import { Range } from '../../../editor/common/core/range.js'; * Base class for all tokens with a `range` that * reflects token position in the original data. */ -export class BaseToken { +export abstract class BaseToken { constructor( public readonly range: Range, ) { } @@ -20,4 +20,20 @@ export class BaseToken { public sameRange(other: Range): boolean { return this.range.equalsRange(other); } + + /** + * Returns a string representation of the token. + */ + public abstract toString(): string; + + /** + * Check if this token is equal to another one. + */ + public equals(other: T): boolean { + if (!(other instanceof this.constructor)) { + return false; + } + + return this.sameRange(other.range); + } } diff --git a/src/vs/editor/common/codecs/linesCodec/tokens/line.ts b/src/vs/editor/common/codecs/linesCodec/tokens/line.ts index eeab82df1633a..6669169967f57 100644 --- a/src/vs/editor/common/codecs/linesCodec/tokens/line.ts +++ b/src/vs/editor/common/codecs/linesCodec/tokens/line.ts @@ -42,8 +42,12 @@ export class Line extends BaseToken { /** * Check if this token is equal to another one. */ - public equals(other: Line): boolean { - if (!super.sameRange(other.range)) { + public override equals(other: T): boolean { + if (!super.equals(other)) { + return false; + } + + if (!(other instanceof Line)) { return false; } diff --git a/src/vs/editor/common/codecs/linesCodec/tokens/newLine.ts b/src/vs/editor/common/codecs/linesCodec/tokens/newLine.ts index 4a60bb9c26e17..e131225bf7f79 100644 --- a/src/vs/editor/common/codecs/linesCodec/tokens/newLine.ts +++ b/src/vs/editor/common/codecs/linesCodec/tokens/newLine.ts @@ -42,13 +42,6 @@ export class NewLine extends BaseToken { ); } - /** - * Check if this token is equal to another one. - */ - public equals(other: NewLine): boolean { - return super.sameRange(other.range); - } - /** * Returns a string representation of the token. */ diff --git a/src/vs/editor/common/codecs/simpleCodec/tokens/space.ts b/src/vs/editor/common/codecs/simpleCodec/tokens/space.ts index aa78aa1755696..9961c38ece9ce 100644 --- a/src/vs/editor/common/codecs/simpleCodec/tokens/space.ts +++ b/src/vs/editor/common/codecs/simpleCodec/tokens/space.ts @@ -37,13 +37,6 @@ export class Space extends BaseToken { )); } - /** - * Check if this token is equal to another one. - */ - public equals(other: Space): boolean { - return super.sameRange(other.range); - } - /** * Returns a string representation of the token. */ diff --git a/src/vs/editor/common/codecs/simpleCodec/tokens/tab.ts b/src/vs/editor/common/codecs/simpleCodec/tokens/tab.ts index 9addc056efb02..aab11327bc156 100644 --- a/src/vs/editor/common/codecs/simpleCodec/tokens/tab.ts +++ b/src/vs/editor/common/codecs/simpleCodec/tokens/tab.ts @@ -38,13 +38,6 @@ export class Tab extends BaseToken { )); } - /** - * Check if this token is equal to another one. - */ - public equals(other: Tab): boolean { - return super.sameRange(other.range); - } - /** * Returns a string representation of the token. */ diff --git a/src/vs/editor/common/codecs/simpleCodec/tokens/word.ts b/src/vs/editor/common/codecs/simpleCodec/tokens/word.ts index efaace782c3af..fc3cefa79be68 100644 --- a/src/vs/editor/common/codecs/simpleCodec/tokens/word.ts +++ b/src/vs/editor/common/codecs/simpleCodec/tokens/word.ts @@ -51,8 +51,12 @@ export class Word extends BaseToken { /** * Check if this token is equal to another one. */ - public equals(other: Word): boolean { - if (!super.sameRange(other.range)) { + public override equals(other: T): boolean { + if (!super.equals(other)) { + return false; + } + + if (!(other instanceof Word)) { return false; } diff --git a/src/vs/editor/test/common/utils/testDecoder.ts b/src/vs/editor/test/common/utils/testDecoder.ts index d8a8fc736c2bb..978ced6ae1c6f 100644 --- a/src/vs/editor/test/common/utils/testDecoder.ts +++ b/src/vs/editor/test/common/utils/testDecoder.ts @@ -10,9 +10,6 @@ import { BaseToken } from '../../../common/codecs/baseToken.js'; import { Disposable } from '../../../../base/common/lifecycle.js'; import { WriteableStream } from '../../../../base/common/stream.js'; import { BaseDecoder } from '../../../../base/common/codecs/baseDecoder.js'; -import { Line, NewLine } from '../../../common/codecs/linesCodec/tokens/index.js'; -import { Space, Tab, Word } from '../../../common/codecs/simpleCodec/tokens/index.js'; -import { FileReference } from '../../../common/codecs/chatbotPromptCodec/tokens/fileReference.js'; /** * (pseudo)Random boolean generator. @@ -99,91 +96,10 @@ export class TestDecoder> extends const expectedToken = expectedTokens[i]; const receivedtoken = receivedTokens[i]; - if (expectedToken instanceof Line) { - assert( - receivedtoken instanceof Line, - `Token '${i}' must be a 'Line', got '${receivedtoken}'.`, - ); - - assert( - receivedtoken.equals(expectedToken), - `Expected token '${i}' to be '${expectedToken}', got '${receivedtoken}'.`, - ); - - continue; - } - - if (expectedToken instanceof NewLine) { - assert( - receivedtoken instanceof NewLine, - `Token '${i}' must be a 'NewLine', got '${receivedtoken}'.`, - ); - - assert( - receivedtoken.equals(expectedToken), - `Expected token '${i}' be '${expectedToken}', got '${receivedtoken}'.`, - ); - - continue; - } - - if (expectedToken instanceof Space) { - assert( - receivedtoken instanceof Space, - `Token '${i}' must be a 'Space', got '${receivedtoken}'.`, - ); - - assert( - receivedtoken.equals(expectedToken), - `Expected token '${i}' be '${expectedToken}', got '${receivedtoken}'.`, - ); - - continue; - } - - if (expectedToken instanceof Word) { - assert( - receivedtoken instanceof Word, - `Token '${i}' must be a 'Word', got '${receivedtoken}'.`, - ); - - assert( - receivedtoken.equals(expectedToken), - `Expected token '${i}' be '${expectedToken}', got '${receivedtoken}'.`, - ); - - continue; - } - - if (expectedToken instanceof Tab) { - assert( - receivedtoken instanceof Tab, - `Token '${i}' must be a 'Tab ', got '${receivedtoken}'.`, - ); - - assert( - receivedtoken.equals(expectedToken), - `Expected token '${i}' be '${expectedToken}', got '${receivedtoken}'.`, - ); - - continue; - } - - if (expectedToken instanceof FileReference) { - assert( - receivedtoken instanceof FileReference, - `Token '${i}' must be a 'FileReference ', got '${receivedtoken}'.`, - ); - - assert( - receivedtoken.equals(expectedToken), - `Expected token '${i}' be '${expectedToken}', got '${receivedtoken}'.`, - ); - - continue; - } - - throw new Error(`Unexpected token type for '${expectedToken}'.`); + assert( + receivedtoken.equals(expectedToken), + `Expected token '${i}' to be '${expectedToken}', got '${receivedtoken}'.`, + ); } assert.strictEqual( diff --git a/src/vs/workbench/contrib/chat/browser/chatInputPart.ts b/src/vs/workbench/contrib/chat/browser/chatInputPart.ts index 17dd8af78ce28..a174d59927e22 100644 --- a/src/vs/workbench/contrib/chat/browser/chatInputPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatInputPart.ts @@ -150,9 +150,12 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge chatWidget: IChatWidget, ): IChatRequestVariableEntry[] { const contextArr = [...this.attachmentModel.attachments]; + if (this._implicitContext?.enabled && this._implicitContext.value) { const mainEntry = this._implicitContext.toBaseEntry(); + contextArr.push(mainEntry); + // if the implicit context is a file, it can have nested // file references that should be included in the context if (this._implicitContext.validFileReferenceUris) { @@ -168,7 +171,6 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge } } - // factor in nested references of dynamic variables into the implicit attached context for (const part of chatWidget.parsedInput.parts) { if (!(part instanceof ChatRequestDynamicVariablePart)) { diff --git a/src/vs/editor/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts b/src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts similarity index 76% rename from src/vs/editor/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts rename to src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts index a0fc79a0e3b95..98ae9f66599e8 100644 --- a/src/vs/editor/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts +++ b/src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/chatbotPromptCodec.ts @@ -3,10 +3,10 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { VSBuffer } from '../../../../base/common/buffer.js'; -import { Disposable } from '../../../../base/common/lifecycle.js'; -import { ReadableStream } from '../../../../base/common/stream.js'; -import { ICodec } from '../../../../base/common/codecs/types/ICodec.js'; +import { VSBuffer } from '../../../../../../base/common/buffer.js'; +import { Disposable } from '../../../../../../base/common/lifecycle.js'; +import { ReadableStream } from '../../../../../../base/common/stream.js'; +import { ICodec } from '../../../../../../base/common/codecs/types/ICodec.js'; import { ChatbotPromptDecoder, TChatbotPromptToken } from './chatbotPromptDecoder.js'; /** diff --git a/src/vs/editor/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts b/src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts similarity index 74% rename from src/vs/editor/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts rename to src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts index 22ad7880d5829..eb2e5c3b840ea 100644 --- a/src/vs/editor/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts +++ b/src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/chatbotPromptDecoder.ts @@ -3,12 +3,12 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Word } from '../simpleCodec/tokens/index.js'; import { FileReference } from './tokens/fileReference.js'; -import { VSBuffer } from '../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../base/common/stream.js'; -import { BaseDecoder } from '../../../../base/common/codecs/baseDecoder.js'; -import { SimpleDecoder, TSimpleToken } from '../simpleCodec/simpleDecoder.js'; +import { VSBuffer } from '../../../../../../base/common/buffer.js'; +import { ReadableStream } from '../../../../../../base/common/stream.js'; +import { BaseDecoder } from '../../../../../../base/common/codecs/baseDecoder.js'; +import { Word } from '../../../../../../editor/common/codecs/simpleCodec/tokens/index.js'; +import { SimpleDecoder, TSimpleToken } from '../../../../../../editor/common/codecs/simpleCodec/simpleDecoder.js'; /** * Tokens handled by the `ChatbotPromptDecoder` decoder. diff --git a/src/vs/editor/common/codecs/chatbotPromptCodec/tokens/fileReference.ts b/src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/tokens/fileReference.ts similarity index 83% rename from src/vs/editor/common/codecs/chatbotPromptCodec/tokens/fileReference.ts rename to src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/tokens/fileReference.ts index d8f32f1060375..a843d8d3517b3 100644 --- a/src/vs/editor/common/codecs/chatbotPromptCodec/tokens/fileReference.ts +++ b/src/vs/workbench/contrib/chat/common/codecs/chatbotPromptCodec/tokens/fileReference.ts @@ -3,10 +3,11 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { BaseToken } from '../../baseToken.js'; -import { Word } from '../../simpleCodec/tokens/index.js'; -import { assert } from '../../../../../base/common/assert.js'; -import { Range } from '../../../../../editor/common/core/range.js'; +import { assert } from '../../../../../../../base/common/assert.js'; +import { Range } from '../../../../../../../editor/common/core/range.js'; +import { BaseToken } from '../../../../../../../editor/common/codecs/baseToken.js'; +import { Word } from '../../../../../../../editor/common/codecs/simpleCodec/tokens/index.js'; + // Start sequence for a file reference token in a prompt. const TOKEN_START: string = '#file:'; @@ -77,11 +78,15 @@ export class FileReference extends BaseToken { /** * Check if this token is equal to another one. */ - public equals(other: FileReference): boolean { + public override equals(other: T): boolean { if (!super.sameRange(other.range)) { return false; } + if (!(other instanceof FileReference)) { + return false; + } + return this.text === other.text; } diff --git a/src/vs/workbench/contrib/chat/common/promptFileReference.ts b/src/vs/workbench/contrib/chat/common/promptFileReference.ts index ac48f7ce29670..7a250dbf9704e 100644 --- a/src/vs/workbench/contrib/chat/common/promptFileReference.ts +++ b/src/vs/workbench/contrib/chat/common/promptFileReference.ts @@ -8,9 +8,9 @@ import { Emitter } from '../../../../base/common/event.js'; import { extUri } from '../../../../base/common/resources.js'; import { Disposable } from '../../../../base/common/lifecycle.js'; import { Location } from '../../../../editor/common/languages.js'; +import { ChatbotPromptCodec } from './codecs/chatbotPromptCodec/chatbotPromptCodec.js'; import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; import { FileOpenFailed, NotPromptSnippetFile, RecursiveReference } from './promptFileReferenceErrors.js'; -import { ChatbotPromptCodec } from '../../../../editor/common/codecs/chatbotPromptCodec/chatbotPromptCodec.js'; import { FileChangesEvent, FileChangeType, IFileService, IFileStreamContent } from '../../../../platform/files/common/files.js'; /** diff --git a/src/vs/editor/test/common/codecs/chatbotPromptCodec.test.ts b/src/vs/workbench/contrib/chat/test/common/codecs/chatbotPromptCodec.test.ts similarity index 87% rename from src/vs/editor/test/common/codecs/chatbotPromptCodec.test.ts rename to src/vs/workbench/contrib/chat/test/common/codecs/chatbotPromptCodec.test.ts index 89f9cc41a63c8..a4a34874a83c0 100644 --- a/src/vs/editor/test/common/codecs/chatbotPromptCodec.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/codecs/chatbotPromptCodec.test.ts @@ -3,13 +3,13 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { TestDecoder } from '../utils/testDecoder.js'; -import { Range } from '../../../common/core/range.js'; -import { VSBuffer } from '../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../base/common/stream.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { VSBuffer } from '../../../../../../base/common/buffer.js'; +import { Range } from '../../../../../../editor/common/core/range.js'; +import { newWriteableStream } from '../../../../../../base/common/stream.js'; +import { TestDecoder } from '../../../../../../editor/test/common/utils/testDecoder.js'; import { FileReference } from '../../../common/codecs/chatbotPromptCodec/tokens/fileReference.js'; import { ChatbotPromptCodec } from '../../../common/codecs/chatbotPromptCodec/chatbotPromptCodec.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; import { ChatbotPromptDecoder, TChatbotPromptToken } from '../../../common/codecs/chatbotPromptCodec/chatbotPromptDecoder.js'; /** diff --git a/src/vs/editor/test/common/codecs/chatbotPromptDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/codecs/chatbotPromptDecoder.test.ts similarity index 86% rename from src/vs/editor/test/common/codecs/chatbotPromptDecoder.test.ts rename to src/vs/workbench/contrib/chat/test/common/codecs/chatbotPromptDecoder.test.ts index de518f475990a..c9d9178574fba 100644 --- a/src/vs/editor/test/common/codecs/chatbotPromptDecoder.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/codecs/chatbotPromptDecoder.test.ts @@ -3,12 +3,12 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { TestDecoder } from '../utils/testDecoder.js'; -import { Range } from '../../../common/core/range.js'; -import { VSBuffer } from '../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../base/common/stream.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { VSBuffer } from '../../../../../../base/common/buffer.js'; +import { Range } from '../../../../../../editor/common/core/range.js'; +import { newWriteableStream } from '../../../../../../base/common/stream.js'; +import { TestDecoder } from '../../../../../../editor/test/common/utils/testDecoder.js'; import { FileReference } from '../../../common/codecs/chatbotPromptCodec/tokens/fileReference.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; import { ChatbotPromptDecoder, TChatbotPromptToken } from '../../../common/codecs/chatbotPromptCodec/chatbotPromptDecoder.js'; /**