From fd0e07d95f9e5a4d277d5f9e6c2936c14f56ae5d Mon Sep 17 00:00:00 2001 From: ivanjoz Date: Fri, 7 Jun 2024 11:17:13 -0500 Subject: [PATCH] improve base94 encode decode responses --- backend/core/helpers.go | 9 +++++++-- backend/core/responses.go | 13 ++++++------ frontend/src/core/halpers.ts | 31 ++++++++++++++++++----------- frontend/src/services/connection.ts | 9 +++++++-- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/backend/core/helpers.go b/backend/core/helpers.go index fa563f2..22a3e55 100644 --- a/backend/core/helpers.go +++ b/backend/core/helpers.go @@ -1371,7 +1371,7 @@ func ConcatInt64(num1, num2 int64) int64 { } // Base94 encoding takes 9 input bytes of 8 bits each, uses those to construct a 72-bit integer, and then converts that to an 11-digit base-94 number, and encodes that number using the ASCII characters ! (33) through ~ (126): -func Base94Encode(data []byte) string { +func Base94EncodeBytes(data []byte) []byte { var encodeChunk = func(chunk []byte) string { // Convert the 9-byte chunk to a 72-bit integer var value big.Int @@ -1414,7 +1414,12 @@ func Base94Encode(data []byte) string { encoded.WriteString(encodeChunk(padded)) } - return fmt.Sprint(padding) + encoded.String() + paddingChar := fmt.Sprint(padding) + return append([]byte(paddingChar), encoded.Bytes()...) +} + +func Base94Encode(data []byte) string { + return string(Base94EncodeBytes(data)) } func Base94Decode(encoded string) ([]byte, error) { diff --git a/backend/core/responses.go b/backend/core/responses.go index 9960906..bf7d7f4 100644 --- a/backend/core/responses.go +++ b/backend/core/responses.go @@ -647,16 +647,17 @@ func (req *HandlerArgs) MakeResponse(respStruct any) HandlerResponse { if err := gz.Close(); err != nil { log.Fatal(err) } + bodyBytes = nil bodyBytesCompressed := bodyCompressed.Bytes() - Log("Body bytes a enviar::", len(bodyBytesCompressed)) + // Convert to base64 string because AWS WebSocket API Gateway does not support binary messages + base94message := Base94EncodeBytes(bodyBytesCompressed) + bodyBytesCompressed = nil + Log("Body bytes a enviar::", len(base94message)) if req.WebSocketConn != nil { - req.WebSocketConn.WriteMessage(websocket.BinaryMessage, bodyBytesCompressed) + req.WebSocketConn.WriteMessage(websocket.TextMessage, base94message) } else if len(req.ConnectionID) > 0 { - // Convert to base64 because AWS WebSocket API Gateway does not support binary messages - bodyBase64Bytes := []byte{} - base64.StdEncoding.Encode(bodyBase64Bytes, bodyBytesCompressed) - SendWebsocketMessage(req.ConnectionID, &bodyBase64Bytes) + SendWebsocketMessage(req.ConnectionID, &base94message) } else { panic("No se pudo enviar el mensaje (Sin ConnID ni Conn)") } diff --git a/frontend/src/core/halpers.ts b/frontend/src/core/halpers.ts index 82a640f..368c663 100644 --- a/frontend/src/core/halpers.ts +++ b/frontend/src/core/halpers.ts @@ -47,11 +47,13 @@ export const base94Encode = (data: Uint8Array): string => { return String(paddingSize) + encoded; } -export const Base94Decode = (encoded: string): Uint8Array => { - const paddingSize = parseInt(encoded[0]); - encoded = encoded.slice(1); - const encodedChunkSize = 11; - const chunkSize = 9; +export const base94Decode = (encoded: string): Uint8Array => { + const paddingSize = parseInt(encoded[0]) + encoded = encoded.slice(1) + const encodedChunkSize = 11 + const chunkSize = 9 + // const binarySize = ((encoded.length / encodedChunkSize) * 9) - paddingSize + // console.log("expected binarySize::", binarySize) const decodeChunk = (chunk: string) => { const asciiOffset = 33; @@ -63,12 +65,12 @@ export const Base94Decode = (encoded: string): Uint8Array => { // Convert the base94 string to a 72-bit integer let value = BigInt(0); for (let i = 0; i < chunk.length; i++) { - let char = BigInt(chunk.charCodeAt(i) - asciiOffset); + const char = BigInt(chunk.charCodeAt(i) - asciiOffset); value = value * BigInt(94) + char; } // Convert the 72-bit integer to a 9-byte chunk - let decoded = []; + const decoded = []; for (let i = chunkSize - 1; i >= 0; i--) { decoded[i] = Number(value & BigInt(0xff)); value >>= BigInt(8); @@ -81,15 +83,20 @@ export const Base94Decode = (encoded: string): Uint8Array => { throw new Error(`Encoded string length must be a multiple of ${encodedChunkSize}`); } - let decoded = []; + const decoded: number[] = []; for (let i = 0; i < encoded.length; i += encodedChunkSize) { - let chunk = encoded.slice(i, i + encodedChunkSize); - let decodedChunk = decodeChunk(chunk); + const chunk = encoded.slice(i, i + encodedChunkSize); + let decodedChunk = decodeChunk(chunk); + if(paddingSize && i === (encoded.length - encodedChunkSize)){ + decodedChunk = decodedChunk.slice(0, 9 - paddingSize) + } decoded.push(...decodedChunk); } - return new Uint8Array(decoded); + const binaryDecoded = new Uint8Array(decoded) + // console.log("final binarySize::", binaryDecoded.length) + return binaryDecoded } export const testBase94encode = () => { @@ -100,7 +107,7 @@ export const testBase94encode = () => { const data = encoder.encode(dataString) const encoded = base94Encode(data) console.log("encoded:", encoded) - const decoded = Base94Decode(encoded) + const decoded = base94Decode(encoded) const decoder = new TextDecoder() const decodedString = decoder.decode(decoded) console.log("decoded:", decodedString) diff --git a/frontend/src/services/connection.ts b/frontend/src/services/connection.ts index 4decad1..c93b1b5 100644 --- a/frontend/src/services/connection.ts +++ b/frontend/src/services/connection.ts @@ -1,7 +1,7 @@ import Dexie from 'dexie' import { createSignal } from 'solid-js' import { GetWssAPI } from '~/app' -import { base94Encode } from '~/core/halpers' +import { base94Decode, base94Encode } from '~/core/halpers' let dexieInitPromise: Promise let dexiedb: Dexie @@ -169,9 +169,14 @@ export class ConnectionManager { }) this.ws.onmessage = (event) => { - const blob = event.data + const base94gzString = event.data + console.log("bytes recibidos por decodificar::", base94gzString.length) + const base94gzArray = base94Decode(base94gzString) + console.log("bytes recibidos decodificados::", base94gzArray.length) + const blob = new Blob([base94gzArray], { type: 'application/gzip' }) const ds = new DecompressionStream('gzip'); const decompressedStream = blob.stream().pipeThrough(ds); + new Response(decompressedStream).blob().then((blob) => { return blob.text() }).then((responseText) => {