Add semantic token struct and serialization (#11)

Semantic tokes are encoded as an array of UInt32 in the LSP spec, which makes sense for transport efficiency. But this is not a convenient representation in the LSP server.

This changeset adds a `SemanticToken` representation, and encode/decode functionality to the transport representation.

It also sort the tokens during encode since the token encoding is offset based with unsigned integers, and must be ordered properly.

Sources/LanguageServerProtocol/LanguageFeatures/SemanticTokens.swift

@@ -73,6 +73,11 @@ public struct SemanticTokensClientCapabilities: Codable, Hashable, Sendable {
 public struct SemanticTokensLegend: Codable, Hashable, Sendable {
     public var tokenTypes: [String]
     public var tokenModifiers: [String]
+
+	public init(tokenTypes: [String], tokenModifiers: [String]) {
+		self.tokenTypes = tokenTypes
+		self.tokenModifiers = tokenModifiers
+	}
 }
 
 public enum SemanticTokenTypes: String, Codable, Hashable, CaseIterable, Sendable {
@@ -131,9 +136,146 @@ public struct SemanticTokensParams: Codable, Hashable, Sendable {
     }
 }
 
+// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#documentSelector
+public struct SemanticToken {
+	// typealias EncodedTuple = (line: UInt32, char: UInt32, length: UInt32, type: UInt32, modifiers: UInt32)
+
+	public let line: UInt32
+	public let char: UInt32
+	public let length: UInt32
+	public let type: UInt32
+	public let modifiers: UInt32
+
+	public static let numFields = 5
+
+	// public func toArray() -> EncodedTuple {
+	// }
+
+	public init(line: UInt32, char: UInt32, length: UInt32, type: UInt32, modifiers: UInt32 = 0) {
+		self.line = line
+		self.char = char
+		self.length = length
+		self.type = type
+		self.modifiers = modifiers
+	}
+}
+
 public struct SemanticTokens: Codable, Hashable, Sendable {
+	/**
+	 * An optional result id. If provided and clients support delta updating
+	 * the client will include the result id in the next semantic token request.
+	 * A server can then instead of computing all semantic tokens again simply
+	 * send a delta.
+	 */
     public var resultId: String?
+
+	/// Encoded token data
     public var data: [UInt32]
+
+
+	public init(resultId: String? = nil, data: [UInt32]) {
+		self.resultId = resultId
+		self.data = data
+	}
+
+	func getLineTokens(_ tokens: Array<SemanticToken>.SubSequence) -> Line {
+		precondition(!tokens.isEmpty)
+
+		var end = tokens.startIndex + 1
+		let line = tokens[tokens.startIndex].line
+
+		while end < tokens.endIndex && tokens[end].line == line {
+			end += 1
+		}
+
+		return Line(line: line, tokens: tokens[tokens.startIndex..<end])
+	}
+
+	mutating func encodeLine(_ tokens: Array<SemanticToken>.SubSequence, prevLine: UInt32) {
+
+		// Sort line tokens
+		let sortedTokens = tokens.sorted { $0.char < $1.char }
+
+		var prevCol: UInt32 = 0
+		var prevLine = prevLine
+
+		for i in 0..<sortedTokens.count {
+			let d0 = (tokens.startIndex + i) * SemanticToken.numFields
+			let t = sortedTokens[i]
+
+			self.data[d0+0] = t.line - prevLine
+			self.data[d0+1] = t.char - prevCol
+			self.data[d0+2] = t.length
+			self.data[d0+3] = t.type
+			self.data[d0+4] = t.modifiers
+
+			prevLine = t.line
+			prevCol = t.char
+		}
+	}
+
+	struct Line {
+		public let line: UInt32
+		public let tokens: Array<SemanticToken>.SubSequence
+	}
+
+	// Convert tokens to encoded packed array format
+	// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens
+	public init(resultId: String? = nil, tokens: [SemanticToken]) {
+		self.resultId = resultId
+		self.data = Array(repeating: 0, count: tokens.count * SemanticToken.numFields)
+
+		var tail = tokens[...]
+		var lines: Array<Line> = []
+		while !tail.isEmpty {
+			let line = getLineTokens(tail)
+			lines.append(line)
+			tail = tail[line.tokens.endIndex...]
+		}
+
+		// Sort lines
+		let sortedLines = lines.sorted { $0.line < $1.line }
+
+		var prevLine: UInt32 = 0
+		for line in sortedLines {
+			encodeLine(line.tokens, prevLine: prevLine)
+			prevLine = line.line
+		}
+	}
+
+	// Convert encoded packed array format to SemanticToken array
+	// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens
+	public func decode() -> [SemanticToken] {
+		var tokens: [SemanticToken] = []
+
+		var currentRow: UInt32 = 0
+		var currentCol: UInt32 = 0
+		let numTokens = data.count/5
+		tokens.reserveCapacity(numTokens)
+
+		for n in 0..<numTokens {
+			let i = n * 5
+
+			// Check if new line
+			if data[i] > 0 {
+				currentCol = 0
+			}
+
+			let token = SemanticToken(
+				line: data[i] + currentRow,
+				char: data[i+1] + currentCol,
+				length: data[i+2],
+				type: data[i+3],
+				modifiers: data[i+4]
+			)
+
+			tokens.append(token)
+			currentRow += data[i]
+			currentCol += data[i+1]
+		}
+
+		return tokens
+	}
 }
 
 public typealias SemanticTokensResponse = SemanticTokens?