From ff6a6d707cd60795deb6a5a13d46400c871f9c06 Mon Sep 17 00:00:00 2001 From: bifrost Date: Thu, 10 Mar 2022 18:15:39 +0100 Subject: [PATCH 01/25] multi-references-for-protobuf --- src/@types.ts | 21 ++- src/AvroHelper.ts | 16 ++ src/JsonHelper.ts | 33 +++- src/ProtoHelper.ts | 34 +++- src/ProtoSchema.ts | 5 + src/SchemaRegistry.spec.ts | 309 ++++++++++++++++++++++++++++++++++++- src/SchemaRegistry.ts | 79 ++++++++-- 7 files changed, 468 insertions(+), 29 deletions(-) diff --git a/src/@types.ts b/src/@types.ts index dc6c8bc..70e758b 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -8,10 +8,15 @@ export enum SchemaType { PROTOBUF = 'PROTOBUF', UNKNOWN = 'UNKNOWN', } - export interface SchemaHelper { validate(schema: Schema): void getSubject(confluentSchema: ConfluentSchema, schema: Schema, separator: string): ConfluentSubject + toConfluentSchema(data: SchemaResponse): ConfluentSchema + getReferences(schema: ConfluentSchema): ReferenceType[] | undefined + updateOptions( + options: ProtocolOptions, + referredSchemas: (string | RawAvroSchema)[], + ): ProtocolOptions } export type AvroOptions = Partial @@ -20,7 +25,7 @@ export type JsonOptions = ConstructorParameters[0] & { compile: (schema: any) => ValidateFunction } } -export type ProtoOptions = { messageName: string } +export type ProtoOptions = { messageName?: string; referredSchemas?: string[] } export interface LegacyOptions { forSchemaOptions?: AvroOptions @@ -62,15 +67,25 @@ export interface AvroConfluentSchema { schema: string | RawAvroSchema } +export type ReferenceType = { + name: string + subject: string + version: number +} export interface ProtoConfluentSchema { type: SchemaType.PROTOBUF schema: string + references?: ReferenceType[] } - export interface JsonConfluentSchema { type: SchemaType.JSON schema: string } +export interface SchemaResponse { + schema: string + schemaType: string + references?: ReferenceType[] +} export type ConfluentSchema = AvroConfluentSchema | ProtoConfluentSchema | JsonConfluentSchema diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index af35ba5..f17e2e9 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -5,9 +5,13 @@ import { ConfluentSchema, SchemaHelper, ConfluentSubject, + ReferenceType, + AvroConfluentSchema, + ProtocolOptions, } from './@types' import { ConfluentSchemaRegistryArgumentError } from './errors' import avro from 'avsc' +import { SchemaResponse, SchemaType } from './@types' export default class AvroHelper implements SchemaHelper { private getRawAvroSchema(schema: ConfluentSchema): RawAvroSchema { @@ -53,4 +57,16 @@ export default class AvroHelper implements SchemaHelper { const asRawAvroSchema = schema as RawAvroSchema return asRawAvroSchema.name != null && asRawAvroSchema.type != null } + + public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + return { type: SchemaType.AVRO, schema: data.schema } + } + + getReferences(_schema: AvroConfluentSchema): ReferenceType[] | undefined { + return undefined + } + + updateOptions(options: ProtocolOptions, _referredSchemas: string[]): ProtocolOptions { + return options + } } diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index 4a4b1cd..a9dff62 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -1,17 +1,38 @@ -// @ts-nocheck -import { Schema, SchemaHelper, ConfluentSubject, ConfluentSchema } from './@types' +import { + Schema, + SchemaHelper, + ConfluentSubject, + ConfluentSchema, + SchemaResponse, + SchemaType, + ReferenceType, + JsonConfluentSchema, + ProtocolOptions, +} from './@types' import { ConfluentSchemaRegistryError } from './errors' export default class JsonHelper implements SchemaHelper { - public validate(schema: Schema): void { + public validate(_schema: Schema): void { return } public getSubject( - confluentSchema: ConfluentSchema, - schema: Schema, - separator: string, + _confluentSchema: ConfluentSchema, + _schema: Schema, + _separator: string, ): ConfluentSubject { throw new ConfluentSchemaRegistryError('not implemented yet') } + + public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + return { type: SchemaType.JSON, schema: data.schema } + } + + getReferences(_schema: JsonConfluentSchema): ReferenceType[] | undefined { + return undefined + } + + updateOptions(options: ProtocolOptions, _referredSchemas: string[]): ProtocolOptions { + return options + } } diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index d71124f..7b1c018 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -1,17 +1,39 @@ -// @ts-nocheck -import { Schema, SchemaHelper, ConfluentSubject, ConfluentSchema } from './@types' +import { + Schema, + SchemaHelper, + ConfluentSubject, + ConfluentSchema, + SchemaResponse, + SchemaType, + ReferenceType, + ProtoConfluentSchema, + ProtocolOptions, +} from './@types' import { ConfluentSchemaRegistryError } from './errors' export default class ProtoHelper implements SchemaHelper { - public validate(schema: Schema): void { + public validate(_schema: Schema): void { return } public getSubject( - confluentSchema: ConfluentSchema, - schema: Schema, - separator: string, + _confluentSchema: ConfluentSchema, + _schema: Schema, + _separator: string, ): ConfluentSubject { throw new ConfluentSchemaRegistryError('not implemented yet') } + + public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + return { type: SchemaType.PROTOBUF, schema: data.schema, references: data.references } + } + + getReferences(schema: ProtoConfluentSchema): ReferenceType[] | undefined { + return schema.references + } + + updateOptions(options: ProtocolOptions, referredSchemas: string[]): ProtocolOptions { + const result = { ...options } + return { ...result, [SchemaType.PROTOBUF]: { ...result[SchemaType.PROTOBUF], referredSchemas } } + } } diff --git a/src/ProtoSchema.ts b/src/ProtoSchema.ts index d69c2d7..23bb3f1 100644 --- a/src/ProtoSchema.ts +++ b/src/ProtoSchema.ts @@ -12,6 +12,11 @@ export default class ProtoSchema implements Schema { constructor(schema: ProtoConfluentSchema, opts?: ProtoOptions) { const parsedMessage = protobuf.parse(schema.schema) const root = parsedMessage.root + + if (opts?.referredSchemas) { + opts.referredSchemas.forEach(rawSchema => protobuf.parse(rawSchema, root)) + } + this.message = root.lookupType(this.getTypeName(parsedMessage, opts)) } diff --git a/src/SchemaRegistry.spec.ts b/src/SchemaRegistry.spec.ts index ac33db4..4208ac7 100644 --- a/src/SchemaRegistry.spec.ts +++ b/src/SchemaRegistry.spec.ts @@ -1,13 +1,14 @@ +/* eslint-disable no-console */ import path from 'path' import { v4 as uuid } from 'uuid' import { readAVSC } from './utils' -import SchemaRegistry from './SchemaRegistry' +import SchemaRegistry, { RegisteredSchema } from './SchemaRegistry' import API, { SchemaRegistryAPIClient } from './api' import { COMPATIBILITY, DEFAULT_API_CLIENT_ID } from './constants' import encodedAnotherPersonV2 from '../fixtures/avro/encodedAnotherPersonV2' import wrongMagicByte from '../fixtures/wrongMagicByte' -import { RawAvroSchema } from './@types' +import { ProtoConfluentSchema, RawAvroSchema, SchemaType } from './@types' const REGISTRY_HOST = 'http://localhost:8982' const schemaRegistryAPIClientArgs = { host: REGISTRY_HOST } @@ -250,3 +251,307 @@ describe('SchemaRegistry - old AVRO api', () => { }) }) }) + +const TestSchemas = { + FirstLevelSchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + + package test; + + import "test/second_level_A.proto"; + import "test/second_level_B.proto"; + + message FirstLevel { + int32 id1 = 1; + SecondLevelA level1a = 2; + SecondLevelB level1b = 3; + }`, + references: [ + { + name: 'test/second_level_A.proto', + subject: 'SecondLevelA', + version: undefined, + }, + { + name: 'test/second_level_B.proto', + subject: 'SecondLevelB', + version: undefined, + }, + ], + } as ProtoConfluentSchema, + + SecondLevelASchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + + package test; + + import "test/third_level.proto"; + + message SecondLevelA { + int32 id2a = 1; + ThirdLevel level2a = 2; + }`, + references: [ + { + name: 'test/third_level.proto', + subject: 'ThirdLevel', + version: undefined, + }, + ], + } as ProtoConfluentSchema, + + SecondLevelBSchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + + package test; + + import "test/third_level.proto"; + + message SecondLevelB { + int32 id2b = 1; + ThirdLevel level2b = 2; + }`, + references: [ + { + name: 'test/third_level.proto', + subject: 'ThirdLevel', + version: undefined, + }, + ], + } as ProtoConfluentSchema, + + ThirdLevelSchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + + package test; + + message ThirdLevel { + int32 id3 = 1; + }`, + } as ProtoConfluentSchema, +} + +function apiResponse(result) { + return JSON.parse(result.responseData) +} + +describe('SchemaRegistry - protobuf', () => { + let schemaRegistry: SchemaRegistry + let registeredSchema: RegisteredSchema + let api + + beforeEach(async () => { + api = API(schemaRegistryAPIClientArgs) + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + }) + + describe('when register', () => { + describe('when no reference', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + }) + it('should return schema id', async () => { + expect(registeredSchema.id).toEqual(expect.any(Number)) + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with reference', () => { + let schemaId + let referenceSchema + + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + schemaId = registeredSchema.id + + const schemaRaw = apiResponse(await api.Schema.find({ id: schemaId })) + referenceSchema = schemaRaw.references[0].subject + }) + + it('should return schema id', async () => { + expect(schemaId).toEqual(expect.any(Number)) + }) + it('should create a schema with reference', async () => { + expect(referenceSchema).toEqual('ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multiple reference', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'FirstLevel', + }) + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) + + describe('_getSchema', () => { + let schema + + describe('no references', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match subject', async () => { + expect(schema.message.name).toEqual('ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with references', () => { + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'ThirdLevel' }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match subject', async () => { + expect(schema.message.name).toEqual('SecondLevelA') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multi references', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'FirstLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match subject', async () => { + expect(schema.message.name).toEqual('FirstLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) +}) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index bc7bb3e..97fb90b 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -21,6 +21,10 @@ import { ConfluentSubject, SchemaRegistryAPIClientOptions, AvroConfluentSchema, + SchemaResponse, + ProtoConfluentSchema, + ProtocolOptions, + SchemaHelper, } from './@types' import { helperTypeFromSchemaType, @@ -28,7 +32,7 @@ import { schemaFromConfluentSchema, } from './schemaTypeResolver' -interface RegisteredSchema { +export interface RegisteredSchema { id: number } @@ -49,7 +53,6 @@ const DEFAULT_OPTS = { compatibility: COMPATIBILITY.BACKWARD, separator: DEFAULT_SEPERATOR, } - export default class SchemaRegistry { private api: SchemaRegistryAPIClient private cacheMissRequests: { [key: number]: Promise } = {} @@ -110,6 +113,8 @@ export default class SchemaRegistry { const confluentSchema: ConfluentSchema = this.getConfluentSchema(schema) const helper = helperTypeFromSchemaType(confluentSchema.type) + + this.options = await this.updateOptions(this.options, confluentSchema) const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) helper.validate(schemaInstance) @@ -141,11 +146,17 @@ export default class SchemaRegistry { } } + const references = + confluentSchema.type === SchemaType.PROTOBUF + ? (confluentSchema as ProtoConfluentSchema).references + : undefined + const response = await this.api.Subject.register({ subject: subject.name, body: { schemaType: confluentSchema.type === SchemaType.AVRO ? undefined : confluentSchema.type, schema: confluentSchema.schema, + references, }, }) @@ -156,6 +167,54 @@ export default class SchemaRegistry { return registeredSchema } + private async updateOptions( + options: SchemaRegistryAPIClientOptions | undefined, + schema: ConfluentSchema, + ) { + const helper = helperTypeFromSchemaType(schema.type) + const referredSchemas = await this.getReferences(schema, helper) + return helper.updateOptions(options as ProtocolOptions, referredSchemas) + } + + private async getReferences( + schema: ConfluentSchema, + helper: SchemaHelper, + ): Promise<(string | RawAvroSchema)[]> { + const referencesSet = new Set() + return this._getReferences(schema, helper, referencesSet) + } + + private async _getReferences( + schema: ConfluentSchema, + helper: SchemaHelper, + referencesSet: Set, + ): Promise<(string | RawAvroSchema)[]> { + let referredSchemas: (string | RawAvroSchema)[] = [] + + const references = helper.getReferences(schema) + if (references) { + for (const reference of references) { + const { name, subject, version } = reference + const key = `${name}-${subject}-${version}` + + if (referencesSet.has(key)) { + continue + } + referencesSet.add(key) + + const versionResponse = await this.api.Subject.version(reference) + const foundSchema = versionResponse.data() as SchemaResponse + + const subSchema = helper.toConfluentSchema(foundSchema) + const subReferredSchemas = await this._getReferences(subSchema, helper, referencesSet) + + referredSchemas.push(subSchema.schema) + referredSchemas = referredSchemas.concat(subReferredSchemas) + } + } + return referredSchemas + } + private async _getSchema( registryId: number, ): Promise<{ type: SchemaType; schema: Schema | AvroSchema }> { @@ -166,18 +225,14 @@ export default class SchemaRegistry { } const response = await this.getSchemaOriginRequest(registryId) - const foundSchema: { schema: string; schemaType: string } = response.data() - const rawSchema = foundSchema.schema + const foundSchema = response.data() as SchemaResponse + const schemaType = schemaTypeFromString(foundSchema.schemaType) - if (schemaType === SchemaType.UNKNOWN) { - throw new ConfluentSchemaRegistryError(`Unknown schema type ${foundSchema.schemaType}`) - } + const helper = helperTypeFromSchemaType(schemaType) + const confluentSchema = helper.toConfluentSchema(foundSchema) - const confluentSchema: ConfluentSchema = { - type: schemaType, - schema: rawSchema, - } + this.options = await this.updateOptions(this.options, confluentSchema) const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) return this.cache.setSchema(registryId, schemaType, schemaInstance) } @@ -294,7 +349,7 @@ export default class SchemaRegistry { return id } - private getSchemaOriginRequest(registryId: number) { + private async getSchemaOriginRequest(registryId: number) { // ensure that cache-misses result in a single origin request if (this.cacheMissRequests[registryId]) { return this.cacheMissRequests[registryId] From 6ee47b073af0fab0c620cfafbaabd5e878d1ed66 Mon Sep 17 00:00:00 2001 From: bifrost Date: Thu, 10 Mar 2022 18:24:01 +0100 Subject: [PATCH 02/25] cleanup --- src/SchemaRegistry.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 97fb90b..18fdca3 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -32,7 +32,7 @@ import { schemaFromConfluentSchema, } from './schemaTypeResolver' -export interface RegisteredSchema { +interface RegisteredSchema { id: number } @@ -349,7 +349,7 @@ export default class SchemaRegistry { return id } - private async getSchemaOriginRequest(registryId: number) { + private getSchemaOriginRequest(registryId: number) { // ensure that cache-misses result in a single origin request if (this.cacheMissRequests[registryId]) { return this.cacheMissRequests[registryId] From cbf39868f9cc08ae42a3fc9b76efd13a7bd7ba75 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 09:12:19 +0100 Subject: [PATCH 03/25] refactoring --- src/@types.ts | 2 +- src/AvroHelper.ts | 7 ++++++- src/JsonHelper.ts | 7 ++++++- src/ProtoHelper.ts | 9 ++++++--- src/SchemaRegistry.ts | 8 ++++---- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/@types.ts b/src/@types.ts index 70e758b..ec85d6a 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -13,7 +13,7 @@ export interface SchemaHelper { getSubject(confluentSchema: ConfluentSchema, schema: Schema, separator: string): ConfluentSubject toConfluentSchema(data: SchemaResponse): ConfluentSchema getReferences(schema: ConfluentSchema): ReferenceType[] | undefined - updateOptions( + updateOptionsFromReferences( options: ProtocolOptions, referredSchemas: (string | RawAvroSchema)[], ): ProtocolOptions diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index f17e2e9..d0ca515 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -63,10 +63,15 @@ export default class AvroHelper implements SchemaHelper { } getReferences(_schema: AvroConfluentSchema): ReferenceType[] | undefined { + // TODO: implement for Avro references return undefined } - updateOptions(options: ProtocolOptions, _referredSchemas: string[]): ProtocolOptions { + updateOptionsFromReferences( + options: ProtocolOptions, + _referredSchemas: (string | RawAvroSchema)[], + ): ProtocolOptions { + // TODO: implement for Avro references return options } } diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index a9dff62..85e4884 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -29,10 +29,15 @@ export default class JsonHelper implements SchemaHelper { } getReferences(_schema: JsonConfluentSchema): ReferenceType[] | undefined { + // TODO: implement for JSON references return undefined } - updateOptions(options: ProtocolOptions, _referredSchemas: string[]): ProtocolOptions { + updateOptionsFromReferences( + options: ProtocolOptions, + _referredSchemas: string[], + ): ProtocolOptions { + // TODO: implement for JSON references return options } } diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index 7b1c018..721d664 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -32,8 +32,11 @@ export default class ProtoHelper implements SchemaHelper { return schema.references } - updateOptions(options: ProtocolOptions, referredSchemas: string[]): ProtocolOptions { - const result = { ...options } - return { ...result, [SchemaType.PROTOBUF]: { ...result[SchemaType.PROTOBUF], referredSchemas } } + updateOptionsFromReferences( + options: ProtocolOptions, + referredSchemas: string[], + ): ProtocolOptions { + const opt = { ...options } + return { ...opt, [SchemaType.PROTOBUF]: { ...opt[SchemaType.PROTOBUF], referredSchemas } } } } diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 18fdca3..120aff4 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -114,7 +114,7 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(confluentSchema.type) - this.options = await this.updateOptions(this.options, confluentSchema) + this.options = await this.updateOptionsFromReferences(this.options, confluentSchema) const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) helper.validate(schemaInstance) @@ -167,13 +167,13 @@ export default class SchemaRegistry { return registeredSchema } - private async updateOptions( + private async updateOptionsFromReferences( options: SchemaRegistryAPIClientOptions | undefined, schema: ConfluentSchema, ) { const helper = helperTypeFromSchemaType(schema.type) const referredSchemas = await this.getReferences(schema, helper) - return helper.updateOptions(options as ProtocolOptions, referredSchemas) + return helper.updateOptionsFromReferences(options as ProtocolOptions, referredSchemas) } private async getReferences( @@ -232,7 +232,7 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(schemaType) const confluentSchema = helper.toConfluentSchema(foundSchema) - this.options = await this.updateOptions(this.options, confluentSchema) + this.options = await this.updateOptionsFromReferences(this.options, confluentSchema) const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) return this.cache.setSchema(registryId, schemaType, schemaInstance) } From 15f59990c5c40d76ec47fe895c2cec12bf2b7b9e Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 09:40:02 +0100 Subject: [PATCH 04/25] refactor --- src/SchemaRegistry.protobuf.spec.ts | 304 +++++++++++++++++++++++++++ src/SchemaRegistry.spec.ts | 308 +--------------------------- src/SchemaRegistry.ts | 2 +- 3 files changed, 307 insertions(+), 307 deletions(-) create mode 100644 src/SchemaRegistry.protobuf.spec.ts diff --git a/src/SchemaRegistry.protobuf.spec.ts b/src/SchemaRegistry.protobuf.spec.ts new file mode 100644 index 0000000..fb645c5 --- /dev/null +++ b/src/SchemaRegistry.protobuf.spec.ts @@ -0,0 +1,304 @@ +import SchemaRegistry, { RegisteredSchema } from './SchemaRegistry' +import API from './api' +import { ProtoConfluentSchema, SchemaType } from './@types' + +const REGISTRY_HOST = 'http://localhost:8982' +const schemaRegistryAPIClientArgs = { host: REGISTRY_HOST } +const schemaRegistryArgs = { host: REGISTRY_HOST } + +const TestSchemas = { + FirstLevelSchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + package test; + import "test/second_level_A.proto"; + import "test/second_level_B.proto"; + + message FirstLevel { + int32 id1 = 1; + SecondLevelA level1a = 2; + SecondLevelB level1b = 3; + }`, + references: [ + { + name: 'test/second_level_A.proto', + subject: 'SecondLevelA', + version: undefined, + }, + { + name: 'test/second_level_B.proto', + subject: 'SecondLevelB', + version: undefined, + }, + ], + } as ProtoConfluentSchema, + + SecondLevelASchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + package test; + import "test/third_level.proto"; + + message SecondLevelA { + int32 id2a = 1; + ThirdLevel level2a = 2; + }`, + references: [ + { + name: 'test/third_level.proto', + subject: 'ThirdLevel', + version: undefined, + }, + ], + } as ProtoConfluentSchema, + + SecondLevelBSchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + package test; + import "test/third_level.proto"; + + message SecondLevelB { + int32 id2b = 1; + ThirdLevel level2b = 2; + }`, + references: [ + { + name: 'test/third_level.proto', + subject: 'ThirdLevel', + version: undefined, + }, + ], + } as ProtoConfluentSchema, + + ThirdLevelSchema: { + type: SchemaType.PROTOBUF, + schema: ` + syntax = "proto3"; + package test; + + message ThirdLevel { + int32 id3 = 1; + }`, + } as ProtoConfluentSchema, +} + +function apiResponse(result) { + return JSON.parse(result.responseData) +} + +describe('SchemaRegistry', () => { + let schemaRegistry: SchemaRegistry + let registeredSchema: RegisteredSchema + let api + + beforeEach(async () => { + api = API(schemaRegistryAPIClientArgs) + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + }) + + describe('when register', () => { + describe('when no reference', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + }) + it('should return schema id', async () => { + expect(registeredSchema.id).toEqual(expect.any(Number)) + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with reference', () => { + let schemaId + let referenceSchema + + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + schemaId = registeredSchema.id + + const schemaRaw = apiResponse(await api.Schema.find({ id: schemaId })) + referenceSchema = schemaRaw.references[0].subject + }) + + it('should return schema id', async () => { + expect(schemaId).toEqual(expect.any(Number)) + }) + it('should create a schema with reference', async () => { + expect(referenceSchema).toEqual('ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multiple reference', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'FirstLevel', + }) + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) + + describe('_getSchema', () => { + let schema + + describe('no references', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match subject', async () => { + expect(schema.message.name).toEqual('ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with references', () => { + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'ThirdLevel' }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match subject', async () => { + expect(schema.message.name).toEqual('SecondLevelA') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multi references', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'FirstLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match subject', async () => { + expect(schema.message.name).toEqual('FirstLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) +}) diff --git a/src/SchemaRegistry.spec.ts b/src/SchemaRegistry.spec.ts index 4208ac7..24c7693 100644 --- a/src/SchemaRegistry.spec.ts +++ b/src/SchemaRegistry.spec.ts @@ -3,12 +3,12 @@ import path from 'path' import { v4 as uuid } from 'uuid' import { readAVSC } from './utils' -import SchemaRegistry, { RegisteredSchema } from './SchemaRegistry' +import SchemaRegistry from './SchemaRegistry' import API, { SchemaRegistryAPIClient } from './api' import { COMPATIBILITY, DEFAULT_API_CLIENT_ID } from './constants' import encodedAnotherPersonV2 from '../fixtures/avro/encodedAnotherPersonV2' import wrongMagicByte from '../fixtures/wrongMagicByte' -import { ProtoConfluentSchema, RawAvroSchema, SchemaType } from './@types' +import { RawAvroSchema } from './@types' const REGISTRY_HOST = 'http://localhost:8982' const schemaRegistryAPIClientArgs = { host: REGISTRY_HOST } @@ -251,307 +251,3 @@ describe('SchemaRegistry - old AVRO api', () => { }) }) }) - -const TestSchemas = { - FirstLevelSchema: { - type: SchemaType.PROTOBUF, - schema: ` - syntax = "proto3"; - - package test; - - import "test/second_level_A.proto"; - import "test/second_level_B.proto"; - - message FirstLevel { - int32 id1 = 1; - SecondLevelA level1a = 2; - SecondLevelB level1b = 3; - }`, - references: [ - { - name: 'test/second_level_A.proto', - subject: 'SecondLevelA', - version: undefined, - }, - { - name: 'test/second_level_B.proto', - subject: 'SecondLevelB', - version: undefined, - }, - ], - } as ProtoConfluentSchema, - - SecondLevelASchema: { - type: SchemaType.PROTOBUF, - schema: ` - syntax = "proto3"; - - package test; - - import "test/third_level.proto"; - - message SecondLevelA { - int32 id2a = 1; - ThirdLevel level2a = 2; - }`, - references: [ - { - name: 'test/third_level.proto', - subject: 'ThirdLevel', - version: undefined, - }, - ], - } as ProtoConfluentSchema, - - SecondLevelBSchema: { - type: SchemaType.PROTOBUF, - schema: ` - syntax = "proto3"; - - package test; - - import "test/third_level.proto"; - - message SecondLevelB { - int32 id2b = 1; - ThirdLevel level2b = 2; - }`, - references: [ - { - name: 'test/third_level.proto', - subject: 'ThirdLevel', - version: undefined, - }, - ], - } as ProtoConfluentSchema, - - ThirdLevelSchema: { - type: SchemaType.PROTOBUF, - schema: ` - syntax = "proto3"; - - package test; - - message ThirdLevel { - int32 id3 = 1; - }`, - } as ProtoConfluentSchema, -} - -function apiResponse(result) { - return JSON.parse(result.responseData) -} - -describe('SchemaRegistry - protobuf', () => { - let schemaRegistry: SchemaRegistry - let registeredSchema: RegisteredSchema - let api - - beforeEach(async () => { - api = API(schemaRegistryAPIClientArgs) - schemaRegistry = new SchemaRegistry(schemaRegistryArgs) - }) - - describe('when register', () => { - describe('when no reference', () => { - beforeEach(async () => { - registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', - }) - }) - it('should return schema id', async () => { - expect(registeredSchema.id).toEqual(expect.any(Number)) - }) - - it('should be able to encode/decode', async () => { - const obj = { id3: 3 } - - const buffer = await schemaRegistry.encode(registeredSchema.id, obj) - const resultObj = await schemaRegistry.decode(buffer) - - expect(resultObj).toEqual(obj) - }) - }) - - describe('with reference', () => { - let schemaId - let referenceSchema - - beforeEach(async () => { - await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', - }) - - const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) - TestSchemas.SecondLevelASchema.references[0].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', - }) - schemaId = registeredSchema.id - - const schemaRaw = apiResponse(await api.Schema.find({ id: schemaId })) - referenceSchema = schemaRaw.references[0].subject - }) - - it('should return schema id', async () => { - expect(schemaId).toEqual(expect.any(Number)) - }) - it('should create a schema with reference', async () => { - expect(referenceSchema).toEqual('ThirdLevel') - }) - - it('should be able to encode/decode', async () => { - const obj = { id2a: 2, level2a: { id3: 3 } } - - const buffer = await schemaRegistry.encode(registeredSchema.id, obj) - const resultObj = await schemaRegistry.decode(buffer) - - expect(resultObj).toEqual(obj) - }) - }) - - describe('with multiple reference', () => { - beforeEach(async () => { - let latest - - await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', - }) - - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) - TestSchemas.SecondLevelASchema.references[0].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', - }) - - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) - TestSchemas.SecondLevelBSchema.references[0].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { - subject: 'SecondLevelB', - }) - - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) - TestSchemas.FirstLevelSchema.references[0].version = latest.version - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) - TestSchemas.FirstLevelSchema.references[1].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { - subject: 'FirstLevel', - }) - }) - - it('should be able to encode/decode', async () => { - const obj = { - id1: 1, - level1a: { id2a: 2, level2a: { id3: 3 } }, - level1b: { id2b: 4, level2b: { id3: 5 } }, - } - - const buffer = await schemaRegistry.encode(registeredSchema.id, obj) - const resultObj = await schemaRegistry.decode(buffer) - - expect(resultObj).toEqual(obj) - }) - }) - }) - - describe('_getSchema', () => { - let schema - - describe('no references', () => { - beforeEach(async () => { - registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', - }) - ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) - }) - - it('should return schema that match subject', async () => { - expect(schema.message.name).toEqual('ThirdLevel') - }) - - it('should be able to encode/decode', async () => { - const obj = { id3: 3 } - - const buffer = await schema.toBuffer(obj) - const resultObj = await schema.fromBuffer(buffer) - - expect(resultObj).toEqual(obj) - }) - }) - - describe('with references', () => { - beforeEach(async () => { - await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'ThirdLevel' }) - - const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) - TestSchemas.SecondLevelASchema.references[0].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', - }) - ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) - }) - - it('should return schema that match subject', async () => { - expect(schema.message.name).toEqual('SecondLevelA') - }) - - it('should be able to encode/decode', async () => { - const obj = { id2a: 2, level2a: { id3: 3 } } - - const buffer = await schema.toBuffer(obj) - const resultObj = await schema.fromBuffer(buffer) - - expect(resultObj).toEqual(obj) - }) - }) - - describe('with multi references', () => { - beforeEach(async () => { - let latest - - await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', - }) - - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) - TestSchemas.SecondLevelASchema.references[0].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', - }) - - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) - TestSchemas.SecondLevelBSchema.references[0].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { - subject: 'SecondLevelB', - }) - - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) - TestSchemas.FirstLevelSchema.references[0].version = latest.version - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) - TestSchemas.FirstLevelSchema.references[1].version = latest.version - registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { - subject: 'FirstLevel', - }) - ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) - }) - - it('should return schema that match subject', async () => { - expect(schema.message.name).toEqual('FirstLevel') - }) - - it('should be able to encode/decode', async () => { - const obj = { - id1: 1, - level1a: { id2a: 2, level2a: { id3: 3 } }, - level1b: { id2b: 4, level2b: { id3: 5 } }, - } - - const buffer = await schema.toBuffer(obj) - const resultObj = await schema.fromBuffer(buffer) - - expect(resultObj).toEqual(obj) - }) - }) - }) -}) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 120aff4..fbbd9e7 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -32,7 +32,7 @@ import { schemaFromConfluentSchema, } from './schemaTypeResolver' -interface RegisteredSchema { +export interface RegisteredSchema { id: number } From 19a1028e764d97008be2fcdb1e4fcc5cf653dfe5 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 13:47:31 +0100 Subject: [PATCH 05/25] add comment --- src/ProtoSchema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ProtoSchema.ts b/src/ProtoSchema.ts index 23bb3f1..350d4b4 100644 --- a/src/ProtoSchema.ts +++ b/src/ProtoSchema.ts @@ -13,6 +13,7 @@ export default class ProtoSchema implements Schema { const parsedMessage = protobuf.parse(schema.schema) const root = parsedMessage.root + // handle all schema references independent on nested references if (opts?.referredSchemas) { opts.referredSchemas.forEach(rawSchema => protobuf.parse(rawSchema, root)) } From 064cfb637c11ed59d67a6817d9a9c1a6e34d6edd Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 13:56:48 +0100 Subject: [PATCH 06/25] bump package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b45e0ef..ad8d74e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@kafkajs/confluent-schema-registry", - "version": "3.2.1", + "version": "3.3.0", "main": "dist/index.js", "description": "ConfluentSchemaRegistry is a library that makes it easier to interact with the Confluent schema registry, it provides convenient methods to encode, decode and register new schemas using the Apache Avro serialization format.", "keywords": [ From 3943fcab1e3a04d7231fc76604cbba613f32abf3 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 14:26:44 +0100 Subject: [PATCH 07/25] rename updateOptionsWithSchemaReferences --- src/@types.ts | 2 +- src/AvroHelper.ts | 3 ++- src/JsonHelper.ts | 3 ++- src/ProtoHelper.ts | 2 +- src/SchemaRegistry.ts | 8 ++++---- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/@types.ts b/src/@types.ts index ec85d6a..39207a4 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -13,7 +13,7 @@ export interface SchemaHelper { getSubject(confluentSchema: ConfluentSchema, schema: Schema, separator: string): ConfluentSubject toConfluentSchema(data: SchemaResponse): ConfluentSchema getReferences(schema: ConfluentSchema): ReferenceType[] | undefined - updateOptionsFromReferences( + updateOptionsFromSchemaReferences( options: ProtocolOptions, referredSchemas: (string | RawAvroSchema)[], ): ProtocolOptions diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index d0ca515..98c5ed6 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -59,6 +59,7 @@ export default class AvroHelper implements SchemaHelper { } public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + // TODO: implement for Avro references return { type: SchemaType.AVRO, schema: data.schema } } @@ -67,7 +68,7 @@ export default class AvroHelper implements SchemaHelper { return undefined } - updateOptionsFromReferences( + updateOptionsFromSchemaReferences( options: ProtocolOptions, _referredSchemas: (string | RawAvroSchema)[], ): ProtocolOptions { diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index 85e4884..85da7a0 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -25,6 +25,7 @@ export default class JsonHelper implements SchemaHelper { } public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + // TODO: implement for JSON references return { type: SchemaType.JSON, schema: data.schema } } @@ -33,7 +34,7 @@ export default class JsonHelper implements SchemaHelper { return undefined } - updateOptionsFromReferences( + updateOptionsFromSchemaReferences( options: ProtocolOptions, _referredSchemas: string[], ): ProtocolOptions { diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index 721d664..052e1c7 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -32,7 +32,7 @@ export default class ProtoHelper implements SchemaHelper { return schema.references } - updateOptionsFromReferences( + updateOptionsFromSchemaReferences( options: ProtocolOptions, referredSchemas: string[], ): ProtocolOptions { diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index fbbd9e7..4d4c092 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -114,7 +114,7 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(confluentSchema.type) - this.options = await this.updateOptionsFromReferences(this.options, confluentSchema) + this.options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) helper.validate(schemaInstance) @@ -167,13 +167,13 @@ export default class SchemaRegistry { return registeredSchema } - private async updateOptionsFromReferences( + private async updateOptionsWithSchemaReferences( options: SchemaRegistryAPIClientOptions | undefined, schema: ConfluentSchema, ) { const helper = helperTypeFromSchemaType(schema.type) const referredSchemas = await this.getReferences(schema, helper) - return helper.updateOptionsFromReferences(options as ProtocolOptions, referredSchemas) + return helper.updateOptionsFromSchemaReferences(options as ProtocolOptions, referredSchemas) } private async getReferences( @@ -232,7 +232,7 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(schemaType) const confluentSchema = helper.toConfluentSchema(foundSchema) - this.options = await this.updateOptionsFromReferences(this.options, confluentSchema) + this.options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) return this.cache.setSchema(registryId, schemaType, schemaInstance) } From b4aefa4d66b8b81abd429559fed404d8583fedd2 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 15:09:41 +0100 Subject: [PATCH 08/25] make requests parallel --- src/SchemaRegistry.ts | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 4d4c092..2ae6e91 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -25,6 +25,7 @@ import { ProtoConfluentSchema, ProtocolOptions, SchemaHelper, + ReferenceType, } from './@types' import { helperTypeFromSchemaType, @@ -189,29 +190,34 @@ export default class SchemaRegistry { helper: SchemaHelper, referencesSet: Set, ): Promise<(string | RawAvroSchema)[]> { - let referredSchemas: (string | RawAvroSchema)[] = [] + const references = helper.getReferences(schema) || [] + // execute in parallel + const schemaPromise = references.map(reference => + this.getReferenceFromReference(reference, helper, referencesSet), + ) + return (await Promise.all(schemaPromise)).flat() + } - const references = helper.getReferences(schema) - if (references) { - for (const reference of references) { - const { name, subject, version } = reference - const key = `${name}-${subject}-${version}` + async getReferenceFromReference( + reference: ReferenceType, + helper: SchemaHelper, + referencesSet: Set, + ): Promise<(string | RawAvroSchema)[]> { + const { name, subject, version } = reference + const key = `${name}-${subject}-${version}` - if (referencesSet.has(key)) { - continue - } - referencesSet.add(key) + if (referencesSet.has(key)) { + return [] + } + referencesSet.add(key) - const versionResponse = await this.api.Subject.version(reference) - const foundSchema = versionResponse.data() as SchemaResponse + const versionResponse = await this.api.Subject.version(reference) + const foundSchema = versionResponse.data() as SchemaResponse - const subSchema = helper.toConfluentSchema(foundSchema) - const subReferredSchemas = await this._getReferences(subSchema, helper, referencesSet) + const subSchema = helper.toConfluentSchema(foundSchema) + const referredSchemas = await this._getReferences(subSchema, helper, referencesSet) - referredSchemas.push(subSchema.schema) - referredSchemas = referredSchemas.concat(subReferredSchemas) - } - } + referredSchemas.push(subSchema.schema) return referredSchemas } From 662b0eee6c33e22d95caecb7030ed7d21bc94bcd Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 15:11:53 +0100 Subject: [PATCH 09/25] rename --- src/SchemaRegistry.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 2ae6e91..850bb77 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -182,10 +182,10 @@ export default class SchemaRegistry { helper: SchemaHelper, ): Promise<(string | RawAvroSchema)[]> { const referencesSet = new Set() - return this._getReferences(schema, helper, referencesSet) + return this.getReferencesRecursive(schema, helper, referencesSet) } - private async _getReferences( + private async getReferencesRecursive( schema: ConfluentSchema, helper: SchemaHelper, referencesSet: Set, @@ -193,12 +193,12 @@ export default class SchemaRegistry { const references = helper.getReferences(schema) || [] // execute in parallel const schemaPromise = references.map(reference => - this.getReferenceFromReference(reference, helper, referencesSet), + this.getReferencesFromReference(reference, helper, referencesSet), ) return (await Promise.all(schemaPromise)).flat() } - async getReferenceFromReference( + async getReferencesFromReference( reference: ReferenceType, helper: SchemaHelper, referencesSet: Set, @@ -215,7 +215,7 @@ export default class SchemaRegistry { const foundSchema = versionResponse.data() as SchemaResponse const subSchema = helper.toConfluentSchema(foundSchema) - const referredSchemas = await this._getReferences(subSchema, helper, referencesSet) + const referredSchemas = await this.getReferencesRecursive(subSchema, helper, referencesSet) referredSchemas.push(subSchema.schema) return referredSchemas From 4ec1311c2963ff10c7d36f4779b57964aa5da8cf Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 15:16:43 +0100 Subject: [PATCH 10/25] rename --- src/SchemaRegistry.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 850bb77..852e904 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -206,6 +206,7 @@ export default class SchemaRegistry { const { name, subject, version } = reference const key = `${name}-${subject}-${version}` + // avoid duplicates if (referencesSet.has(key)) { return [] } @@ -214,10 +215,10 @@ export default class SchemaRegistry { const versionResponse = await this.api.Subject.version(reference) const foundSchema = versionResponse.data() as SchemaResponse - const subSchema = helper.toConfluentSchema(foundSchema) - const referredSchemas = await this.getReferencesRecursive(subSchema, helper, referencesSet) + const schema = helper.toConfluentSchema(foundSchema) + const referredSchemas = await this.getReferencesRecursive(schema, helper, referencesSet) - referredSchemas.push(subSchema.schema) + referredSchemas.push(schema.schema) return referredSchemas } From 54c62185d68ba287cb29ce78f296bf99078937e7 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 15:21:04 +0100 Subject: [PATCH 11/25] rename --- src/SchemaRegistry.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 852e904..cc2249f 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -173,19 +173,19 @@ export default class SchemaRegistry { schema: ConfluentSchema, ) { const helper = helperTypeFromSchemaType(schema.type) - const referredSchemas = await this.getReferences(schema, helper) + const referredSchemas = await this.getReferredSchemas(schema, helper) return helper.updateOptionsFromSchemaReferences(options as ProtocolOptions, referredSchemas) } - private async getReferences( + private async getReferredSchemas( schema: ConfluentSchema, helper: SchemaHelper, ): Promise<(string | RawAvroSchema)[]> { const referencesSet = new Set() - return this.getReferencesRecursive(schema, helper, referencesSet) + return this.getReferredSchemasRecursive(schema, helper, referencesSet) } - private async getReferencesRecursive( + private async getReferredSchemasRecursive( schema: ConfluentSchema, helper: SchemaHelper, referencesSet: Set, @@ -193,12 +193,12 @@ export default class SchemaRegistry { const references = helper.getReferences(schema) || [] // execute in parallel const schemaPromise = references.map(reference => - this.getReferencesFromReference(reference, helper, referencesSet), + this.getReferredSchemasFromReference(reference, helper, referencesSet), ) return (await Promise.all(schemaPromise)).flat() } - async getReferencesFromReference( + async getReferredSchemasFromReference( reference: ReferenceType, helper: SchemaHelper, referencesSet: Set, @@ -216,7 +216,7 @@ export default class SchemaRegistry { const foundSchema = versionResponse.data() as SchemaResponse const schema = helper.toConfluentSchema(foundSchema) - const referredSchemas = await this.getReferencesRecursive(schema, helper, referencesSet) + const referredSchemas = await this.getReferredSchemasRecursive(schema, helper, referencesSet) referredSchemas.push(schema.schema) return referredSchemas From 00c1aaa47e767f8694b57706e8b5c42a1cba6a55 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 15:24:01 +0100 Subject: [PATCH 12/25] remove /* eslint-disable no-console */ --- src/SchemaRegistry.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/SchemaRegistry.spec.ts b/src/SchemaRegistry.spec.ts index 24c7693..ac33db4 100644 --- a/src/SchemaRegistry.spec.ts +++ b/src/SchemaRegistry.spec.ts @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ import path from 'path' import { v4 as uuid } from 'uuid' From 97ceaeb8c208528574f2550ebbe76e268d76dca4 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 11 Mar 2022 19:19:56 +0100 Subject: [PATCH 13/25] add nested references for json --- src/@types.ts | 2 + src/JsonHelper.ts | 14 +- src/JsonSchema.ts | 8 +- src/SchemaRegistry.json.spec.ts | 297 ++++++++++++++++++++++++++++ src/SchemaRegistry.protobuf.spec.ts | 64 +++--- src/SchemaRegistry.ts | 2 +- 6 files changed, 345 insertions(+), 42 deletions(-) create mode 100644 src/SchemaRegistry.json.spec.ts diff --git a/src/@types.ts b/src/@types.ts index 39207a4..1283a3e 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -24,6 +24,7 @@ export type JsonOptions = ConstructorParameters[0] & { ajvInstance?: { compile: (schema: any) => ValidateFunction } + referredSchemas?: string[] } export type ProtoOptions = { messageName?: string; referredSchemas?: string[] } @@ -80,6 +81,7 @@ export interface ProtoConfluentSchema { export interface JsonConfluentSchema { type: SchemaType.JSON schema: string + references?: ReferenceType[] } export interface SchemaResponse { schema: string diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index 85da7a0..6f8b0ed 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -25,20 +25,18 @@ export default class JsonHelper implements SchemaHelper { } public toConfluentSchema(data: SchemaResponse): ConfluentSchema { - // TODO: implement for JSON references - return { type: SchemaType.JSON, schema: data.schema } + return { type: SchemaType.JSON, schema: data.schema, references: data.references } } - getReferences(_schema: JsonConfluentSchema): ReferenceType[] | undefined { - // TODO: implement for JSON references - return undefined + getReferences(schema: JsonConfluentSchema): ReferenceType[] | undefined { + return schema.references } updateOptionsFromSchemaReferences( options: ProtocolOptions, - _referredSchemas: string[], + referredSchemas: string[], ): ProtocolOptions { - // TODO: implement for JSON references - return options + const opt = { ...options } + return { ...opt, [SchemaType.JSON]: { ...opt[SchemaType.JSON], referredSchemas } } } } diff --git a/src/JsonSchema.ts b/src/JsonSchema.ts index a2acc35..bfdc151 100644 --- a/src/JsonSchema.ts +++ b/src/JsonSchema.ts @@ -6,7 +6,6 @@ interface BaseAjvValidationError { data?: unknown schema?: unknown } - interface OldAjvValidationError extends BaseAjvValidationError { dataPath: string instancePath?: string @@ -30,6 +29,13 @@ export default class JsonSchema implements Schema { private getJsonSchema(schema: JsonConfluentSchema, opts?: JsonOptions) { const ajv = opts?.ajvInstance ?? new Ajv(opts) + if (opts?.referredSchemas) { + opts.referredSchemas.forEach(rawSchema => { + const $schema = JSON.parse(rawSchema) + // @ts-ignore + ajv.addSchema($schema, $schema['$id']) + }) + } const validate = ajv.compile(JSON.parse(schema.schema)) return validate } diff --git a/src/SchemaRegistry.json.spec.ts b/src/SchemaRegistry.json.spec.ts new file mode 100644 index 0000000..e11070e --- /dev/null +++ b/src/SchemaRegistry.json.spec.ts @@ -0,0 +1,297 @@ +import SchemaRegistry, { RegisteredSchema } from './SchemaRegistry' +import API from './api' +import { JsonConfluentSchema, SchemaType } from './@types' + +const REGISTRY_HOST = 'http://localhost:8982' +const schemaRegistryAPIClientArgs = { host: REGISTRY_HOST } +const schemaRegistryArgs = { host: REGISTRY_HOST } + +const TestSchemas = { + ThirdLevelSchema: { + type: SchemaType.JSON, + schema: ` + { + "$id": "https://sumup.com/schemas/ThirdLevel", + "type": "object", + "properties": { + "id3": { "type": "number" } + } + } + `, + } as JsonConfluentSchema, + + SecondLevelASchema: { + type: SchemaType.JSON, + schema: ` + { + "$id": "https://sumup.com/schemas/SecondLevelA", + "type": "object", + "properties": { + "id2a": { "type": "number" }, + "level2a": { "$ref": "https://sumup.com/schemas/ThirdLevel" } + } + } + `, + references: [ + { + name: 'https://sumup.com/schemas/ThirdLevel', + subject: 'JSON:ThirdLevel', + version: undefined, + }, + ], + } as JsonConfluentSchema, + + SecondLevelBSchema: { + type: SchemaType.JSON, + schema: ` + { + "$id": "https://sumup.com/schemas/SecondLevelB", + "type": "object", + "properties": { + "id2b": { "type": "number" }, + "level2b": { "$ref": "https://sumup.com/schemas/ThirdLevel" } + } + } + `, + references: [ + { + name: 'https://sumup.com/schemas/ThirdLevel', + subject: 'JSON:ThirdLevel', + version: undefined, + }, + ], + } as JsonConfluentSchema, + + FirstLevelSchema: { + type: SchemaType.JSON, + schema: ` + { + "$id": "https://sumup.com/schemas/FirstLevel", + "type": "object", + "properties": { + "id1": { "type": "number" }, + "level1a": { "$ref": "https://sumup.com/schemas/SecondLevelA" }, + "level1b": { "$ref": "https://sumup.com/schemas/SecondLevelB" } + } + } + `, + references: [ + { + name: 'https://sumup.com/schemas/SecondLevelA', + subject: 'JSON:SecondLevelA', + version: undefined, + }, + { + name: 'https://sumup.com/schemas/SecondLevelB', + subject: 'JSON:SecondLevelB', + version: undefined, + }, + ], + } as JsonConfluentSchema, +} + +function apiResponse(result) { + return JSON.parse(result.responseData) +} + +describe('SchemaRegistry', () => { + let schemaRegistry: SchemaRegistry + let registeredSchema: RegisteredSchema + let api + + beforeEach(async () => { + api = API(schemaRegistryAPIClientArgs) + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + }) + + describe('when register', () => { + describe('when no reference', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'JSON:ThirdLevel', + }) + }) + it('should return schema id', async () => { + expect(registeredSchema.id).toEqual(expect.any(Number)) + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with reference', () => { + let schemaId + let referenceSchema + + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'JSON:ThirdLevel', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'JSON:SecondLevelA', + }) + schemaId = registeredSchema.id + + const schemaRaw = apiResponse(await api.Schema.find({ id: schemaId })) + referenceSchema = schemaRaw.references[0].subject + }) + + it('should return schema id', async () => { + expect(schemaId).toEqual(expect.any(Number)) + }) + + it('should create a schema with reference', async () => { + expect(referenceSchema).toEqual('JSON:ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multiple reference', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'JSON:ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'JSON:SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'JSON:SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'JSON:FirstLevel', + }) + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) + + describe('_getSchema', () => { + let schema + + describe('no references', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'JSON:ThirdLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with references', () => { + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'JSON:ThirdLevel' }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'JSON:SecondLevelA', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multi references', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'JSON:ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'JSON:SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'JSON:SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'JSON:FirstLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) +}) diff --git a/src/SchemaRegistry.protobuf.spec.ts b/src/SchemaRegistry.protobuf.spec.ts index fb645c5..35dc6e5 100644 --- a/src/SchemaRegistry.protobuf.spec.ts +++ b/src/SchemaRegistry.protobuf.spec.ts @@ -23,12 +23,12 @@ const TestSchemas = { references: [ { name: 'test/second_level_A.proto', - subject: 'SecondLevelA', + subject: 'Proto:SecondLevelA', version: undefined, }, { name: 'test/second_level_B.proto', - subject: 'SecondLevelB', + subject: 'Proto:SecondLevelB', version: undefined, }, ], @@ -48,7 +48,7 @@ const TestSchemas = { references: [ { name: 'test/third_level.proto', - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', version: undefined, }, ], @@ -68,7 +68,7 @@ const TestSchemas = { references: [ { name: 'test/third_level.proto', - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', version: undefined, }, ], @@ -104,7 +104,7 @@ describe('SchemaRegistry', () => { describe('when no reference', () => { beforeEach(async () => { registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', }) }) it('should return schema id', async () => { @@ -127,13 +127,13 @@ describe('SchemaRegistry', () => { beforeEach(async () => { await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', }) - const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:ThirdLevel' })) TestSchemas.SecondLevelASchema.references[0].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', + subject: 'Proto:SecondLevelA', }) schemaId = registeredSchema.id @@ -145,7 +145,7 @@ describe('SchemaRegistry', () => { expect(schemaId).toEqual(expect.any(Number)) }) it('should create a schema with reference', async () => { - expect(referenceSchema).toEqual('ThirdLevel') + expect(referenceSchema).toEqual('Proto:ThirdLevel') }) it('should be able to encode/decode', async () => { @@ -163,27 +163,27 @@ describe('SchemaRegistry', () => { let latest await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', }) - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:ThirdLevel' })) TestSchemas.SecondLevelASchema.references[0].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', + subject: 'Proto:SecondLevelA', }) - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:ThirdLevel' })) TestSchemas.SecondLevelBSchema.references[0].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { - subject: 'SecondLevelB', + subject: 'Proto:SecondLevelB', }) - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:SecondLevelA' })) TestSchemas.FirstLevelSchema.references[0].version = latest.version - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:SecondLevelB' })) TestSchemas.FirstLevelSchema.references[1].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { - subject: 'FirstLevel', + subject: 'Proto:FirstLevel', }) }) @@ -208,12 +208,12 @@ describe('SchemaRegistry', () => { describe('no references', () => { beforeEach(async () => { registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', }) ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) }) - it('should return schema that match subject', async () => { + it('should return schema that match message', async () => { expect(schema.message.name).toEqual('ThirdLevel') }) @@ -229,17 +229,17 @@ describe('SchemaRegistry', () => { describe('with references', () => { beforeEach(async () => { - await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'ThirdLevel' }) + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'Proto:ThirdLevel' }) - const latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:ThirdLevel' })) TestSchemas.SecondLevelASchema.references[0].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', + subject: 'Proto:SecondLevelA', }) ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) }) - it('should return schema that match subject', async () => { + it('should return schema that match message', async () => { expect(schema.message.name).toEqual('SecondLevelA') }) @@ -258,32 +258,32 @@ describe('SchemaRegistry', () => { let latest await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { - subject: 'ThirdLevel', + subject: 'Proto:ThirdLevel', }) - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:ThirdLevel' })) TestSchemas.SecondLevelASchema.references[0].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { - subject: 'SecondLevelA', + subject: 'Proto:SecondLevelA', }) - latest = apiResponse(await api.Subject.latestVersion({ subject: 'ThirdLevel' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:ThirdLevel' })) TestSchemas.SecondLevelBSchema.references[0].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { - subject: 'SecondLevelB', + subject: 'Proto:SecondLevelB', }) - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelA' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:SecondLevelA' })) TestSchemas.FirstLevelSchema.references[0].version = latest.version - latest = apiResponse(await api.Subject.latestVersion({ subject: 'SecondLevelB' })) + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:SecondLevelB' })) TestSchemas.FirstLevelSchema.references[1].version = latest.version registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { - subject: 'FirstLevel', + subject: 'Proto:FirstLevel', }) ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) }) - it('should return schema that match subject', async () => { + it('should return schema that match message', async () => { expect(schema.message.name).toEqual('FirstLevel') }) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index cc2249f..866bb39 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -148,7 +148,7 @@ export default class SchemaRegistry { } const references = - confluentSchema.type === SchemaType.PROTOBUF + confluentSchema.type === SchemaType.PROTOBUF || confluentSchema.type === SchemaType.JSON ? (confluentSchema as ProtoConfluentSchema).references : undefined From d2bab57f7024dbedd5cda9e1d19de81e841c58fc Mon Sep 17 00:00:00 2001 From: bifrost Date: Mon, 14 Mar 2022 15:22:18 +0100 Subject: [PATCH 14/25] add documentation for nested references --- docs/schema-json.md | 74 +++++++++++++++++++++++++++++ docs/schema-protobuf.md | 72 ++++++++++++++++++++++++++++ src/SchemaRegistry.json.spec.ts | 48 +++++++++++++++++++ src/SchemaRegistry.protobuf.spec.ts | 49 +++++++++++++++++++ 4 files changed, 243 insertions(+) create mode 100755 docs/schema-json.md create mode 100755 docs/schema-protobuf.md diff --git a/docs/schema-json.md b/docs/schema-json.md new file mode 100755 index 0000000..f3cae98 --- /dev/null +++ b/docs/schema-json.md @@ -0,0 +1,74 @@ +--- +id: schema-json +title: Example JSON Schemas +sidebar_label: Example JSON Schemas +--- + +## Schema with references to other schemas + +You might want to split the JSON definition into several schemas one for each type. + +```JSON +{ + "$id": "https://sumup.com/schemas/A", + "type": "object", + "properties": { + "id": { "type": "number" }, + "b": { "$ref": "https://sumup.com/schemas/B" } + } +} +``` + +```JSON +{ + "$id": "https://sumup.com/schemas/B", + "type": "object", + "properties": { + "id": { "type": "number" } + } +} +``` + +To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the import statement, a schema `subject` and a schema `version`. + +Notice the library will handle an arbitrary number of nested levels. + +```js +const schemaA = ` +{ + "$id": "https://sumup.com/schemas/A", + "type": "object", + "properties": { + "id": { "type": "number" }, + "b": { "$ref": "https://sumup.com/schemas/B" } + } +}` + +const schemaB = ` +{ + "$id": "https://sumup.com/schemas/B", + "type": "object", + "properties": { + "id": { "type": "number" } + } +}` + +await schemaRegistry.register( + { type: SchemaType.JSON, schema: schemaB }, + { subject: 'B'}) + +const { id } = await schemaRegistry.register( + { type: SchemaType.JSON, schema: schemaA, references: [ + { + name: 'https://sumup.com/schemas/B', + subject: 'B', + version: 1, + }, + ]}, + { subject: 'A' }) + +const obj = { id: 1, b: { id: 2 } } + +const buffer = await schemaRegistry.encode(id, obj) +const decodedObj = await schemaRegistry.decode(buffer) +``` diff --git a/docs/schema-protobuf.md b/docs/schema-protobuf.md new file mode 100755 index 0000000..2e4a976 --- /dev/null +++ b/docs/schema-protobuf.md @@ -0,0 +1,72 @@ +--- +id: schema-protobuf +title: Example Protobuf Schemas +sidebar_label: Example Protobuf Schemas +--- + +## Schema with references to other schemas + +You might want to split the Protobuf definition into several schemas one for each type. + +```protobuf +syntax = "proto3"; +package test; +import "test/B.proto"; + +message A { + int32 id = 1; + B b = 2; +} +``` + +```protobuf +syntax = "proto3"; +package test; + +message B { + int32 id = 1; +} +``` + +To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the import statement, a schema `subject` and a schema `version`. + +Notice the library will handle an arbitrary number of nested levels. + +```js +const schemaA = ` + syntax = "proto3"; + package test; + import "test/B.proto"; + + message A { + int32 id = 1; + B b = 2; + }` + +const schemaB = ` + syntax = "proto3"; + package test; + + message B { + int32 id = 1; + }` + +await schemaRegistry.register( + { type: SchemaType.PROTOBUF, schema: schemaB }, + { subject: 'B'}) + +const { id } = await schemaRegistry.register( + { type: SchemaType.PROTOBUF, schema: schemaA, references: [ + { + name: 'test/B.proto', + subject: 'B', + version: 1, + }, + ]}, + { subject: 'A' }) + +const obj = { id: 1, b: { id: 2 } } + +const buffer = await schemaRegistry.encode(id, obj) +const decodedObj = await schemaRegistry.decode(buffer) +``` diff --git a/src/SchemaRegistry.json.spec.ts b/src/SchemaRegistry.json.spec.ts index e11070e..1b3bd9d 100644 --- a/src/SchemaRegistry.json.spec.ts +++ b/src/SchemaRegistry.json.spec.ts @@ -294,4 +294,52 @@ describe('SchemaRegistry', () => { }) }) }) + + describe('when document example', async () => { + it('should encode/decode', async () => { + const schemaA = ` + { + "$id": "https://sumup.com/schemas/A", + "type": "object", + "properties": { + "id": { "type": "number" }, + "b": { "$ref": "https://sumup.com/schemas/B" } + } + }` + + const schemaB = ` + { + "$id": "https://sumup.com/schemas/B", + "type": "object", + "properties": { + "id": { "type": "number" } + } + }` + + await schemaRegistry.register( + { type: SchemaType.JSON, schema: schemaB }, + { subject: 'JSON:B' }, + ) + + const { id } = await schemaRegistry.register( + { + type: SchemaType.JSON, + schema: schemaA, + references: [ + { + name: 'https://sumup.com/schemas/B', + subject: 'JSON:B', + version: 1, + }, + ], + }, + { subject: 'JSON:A' }, + ) + + const obj = { id: 1, b: { id: 2 } } + + const buffer = await schemaRegistry.encode(id, obj) + const decodedObj = await schemaRegistry.decode(buffer) + }) + }) }) diff --git a/src/SchemaRegistry.protobuf.spec.ts b/src/SchemaRegistry.protobuf.spec.ts index 35dc6e5..2b3d80b 100644 --- a/src/SchemaRegistry.protobuf.spec.ts +++ b/src/SchemaRegistry.protobuf.spec.ts @@ -301,4 +301,53 @@ describe('SchemaRegistry', () => { }) }) }) + + describe('when document example', async () => { + it('should encode/decode', async () => { + const schemaA = ` + syntax = "proto3"; + package test; + import "test/B.proto"; + + message A { + int32 id = 1; + B b = 2; + }` + + const schemaB = ` + syntax = "proto3"; + package test; + + message B { + int32 id = 1; + }` + + await schemaRegistry.register( + { type: SchemaType.PROTOBUF, schema: schemaB }, + { subject: 'Proto:B' }, + ) + + const { id } = await schemaRegistry.register( + { + type: SchemaType.PROTOBUF, + schema: schemaA, + references: [ + { + name: 'test/B.proto', + subject: 'Proto:B', + version: 1, + }, + ], + }, + { subject: 'Proto:A' }, + ) + + const obj = { id: 1, b: { id: 2 } } + + const buffer = await schemaRegistry.encode(id, obj) + const decodedObj = await schemaRegistry.decode(buffer) + + expect(decodedObj).toEqual(obj) + }) + }) }) From bc01ead116621ea2c09cdc168f35a62c3b1ae65c Mon Sep 17 00:00:00 2001 From: bifrost Date: Tue, 15 Mar 2022 13:29:39 +0100 Subject: [PATCH 15/25] add-nested-references-for-avro --- docs/schema-json.md | 4 +- docs/schema-protobuf.md | 4 +- src/@types.ts | 6 +- src/AvroHelper.ts | 39 ++- src/JsonHelper.ts | 8 +- src/ProtoHelper.ts | 8 +- src/SchemaRegistry.avro.spec.ts | 380 ++++++++++++++++++++++++++++ src/SchemaRegistry.json.spec.ts | 22 +- src/SchemaRegistry.protobuf.spec.ts | 22 +- src/SchemaRegistry.ts | 30 +-- 10 files changed, 474 insertions(+), 49 deletions(-) create mode 100644 src/SchemaRegistry.avro.spec.ts diff --git a/docs/schema-json.md b/docs/schema-json.md index f3cae98..b3b8164 100755 --- a/docs/schema-json.md +++ b/docs/schema-json.md @@ -57,12 +57,14 @@ await schemaRegistry.register( { type: SchemaType.JSON, schema: schemaB }, { subject: 'B'}) +const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'B' })) + const { id } = await schemaRegistry.register( { type: SchemaType.JSON, schema: schemaA, references: [ { name: 'https://sumup.com/schemas/B', subject: 'B', - version: 1, + version, }, ]}, { subject: 'A' }) diff --git a/docs/schema-protobuf.md b/docs/schema-protobuf.md index 2e4a976..9f74c03 100755 --- a/docs/schema-protobuf.md +++ b/docs/schema-protobuf.md @@ -55,12 +55,14 @@ await schemaRegistry.register( { type: SchemaType.PROTOBUF, schema: schemaB }, { subject: 'B'}) +const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'B' })) + const { id } = await schemaRegistry.register( { type: SchemaType.PROTOBUF, schema: schemaA, references: [ { name: 'test/B.proto', subject: 'B', - version: 1, + version, }, ]}, { subject: 'A' }) diff --git a/src/@types.ts b/src/@types.ts index 1283a3e..ef4b297 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -12,14 +12,15 @@ export interface SchemaHelper { validate(schema: Schema): void getSubject(confluentSchema: ConfluentSchema, schema: Schema, separator: string): ConfluentSubject toConfluentSchema(data: SchemaResponse): ConfluentSchema - getReferences(schema: ConfluentSchema): ReferenceType[] | undefined updateOptionsFromSchemaReferences( options: ProtocolOptions, referredSchemas: (string | RawAvroSchema)[], ): ProtocolOptions } -export type AvroOptions = Partial +export type AvroOptions = Partial & { + referredSchemas?: (string | RawAvroSchema)[] +} export type JsonOptions = ConstructorParameters[0] & { ajvInstance?: { compile: (schema: any) => ValidateFunction @@ -66,6 +67,7 @@ export interface ConfluentSubject { export interface AvroConfluentSchema { type: SchemaType.AVRO schema: string | RawAvroSchema + references?: ReferenceType[] } export type ReferenceType = { diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index 98c5ed6..72bb0d8 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -10,7 +10,7 @@ import { ProtocolOptions, } from './@types' import { ConfluentSchemaRegistryArgumentError } from './errors' -import avro from 'avsc' +import avro, { ForSchemaOptions } from 'avsc' import { SchemaResponse, SchemaType } from './@types' export default class AvroHelper implements SchemaHelper { @@ -25,7 +25,28 @@ export default class AvroHelper implements SchemaHelper { ? schema : this.getRawAvroSchema(schema) // @ts-ignore TODO: Fix typings for Schema... - const avroSchema: AvroSchema = avro.Type.forSchema(rawSchema, opts) + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const me = this + const avroSchema = avro.Type.forSchema(rawSchema, { + ...opts, + // @ts-ignore + typeHook: + opts?.typeHook || + function(_schema: avro.Schema, opts: ForSchemaOptions) { + const avroOpts = opts as AvroOptions + avroOpts?.referredSchemas?.forEach(subSchema => { + const confluentSchema = { + schema: subSchema, + type: SchemaType.AVRO, + } as ConfluentSchema + const rawSubSchema = me.getRawAvroSchema(confluentSchema) + avroOpts.typeHook = undefined + avro.Type.forSchema(rawSubSchema, avroOpts) + }) + }, + }) + return avroSchema } @@ -59,20 +80,14 @@ export default class AvroHelper implements SchemaHelper { } public toConfluentSchema(data: SchemaResponse): ConfluentSchema { - // TODO: implement for Avro references - return { type: SchemaType.AVRO, schema: data.schema } - } - - getReferences(_schema: AvroConfluentSchema): ReferenceType[] | undefined { - // TODO: implement for Avro references - return undefined + return { type: SchemaType.AVRO, schema: data.schema, references: data.references } } updateOptionsFromSchemaReferences( options: ProtocolOptions, - _referredSchemas: (string | RawAvroSchema)[], + referredSchemas: (string | RawAvroSchema)[], ): ProtocolOptions { - // TODO: implement for Avro references - return options + const opts = options ?? {} + return { ...opts, [SchemaType.AVRO]: { ...opts[SchemaType.AVRO], referredSchemas } } } } diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index 6f8b0ed..7493ad0 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -28,15 +28,11 @@ export default class JsonHelper implements SchemaHelper { return { type: SchemaType.JSON, schema: data.schema, references: data.references } } - getReferences(schema: JsonConfluentSchema): ReferenceType[] | undefined { - return schema.references - } - updateOptionsFromSchemaReferences( options: ProtocolOptions, referredSchemas: string[], ): ProtocolOptions { - const opt = { ...options } - return { ...opt, [SchemaType.JSON]: { ...opt[SchemaType.JSON], referredSchemas } } + const opts = options ?? {} + return { ...opts, [SchemaType.JSON]: { ...opts[SchemaType.JSON], referredSchemas } } } } diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index 052e1c7..3c52a51 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -28,15 +28,11 @@ export default class ProtoHelper implements SchemaHelper { return { type: SchemaType.PROTOBUF, schema: data.schema, references: data.references } } - getReferences(schema: ProtoConfluentSchema): ReferenceType[] | undefined { - return schema.references - } - updateOptionsFromSchemaReferences( options: ProtocolOptions, referredSchemas: string[], ): ProtocolOptions { - const opt = { ...options } - return { ...opt, [SchemaType.PROTOBUF]: { ...opt[SchemaType.PROTOBUF], referredSchemas } } + const opts = options ?? {} + return { ...opts, [SchemaType.PROTOBUF]: { ...opts[SchemaType.PROTOBUF], referredSchemas } } } } diff --git a/src/SchemaRegistry.avro.spec.ts b/src/SchemaRegistry.avro.spec.ts new file mode 100644 index 0000000..64812b3 --- /dev/null +++ b/src/SchemaRegistry.avro.spec.ts @@ -0,0 +1,380 @@ +import SchemaRegistry, { RegisteredSchema } from './SchemaRegistry' +import API from './api' +import { AvroConfluentSchema, SchemaType } from './@types' + +const REGISTRY_HOST = 'http://localhost:8982' +const schemaRegistryAPIClientArgs = { host: REGISTRY_HOST } +const schemaRegistryArgs = { host: REGISTRY_HOST } + +const TestSchemas = { + FirstLevelSchema: { + type: SchemaType.AVRO, + schema: ` + { + "type" : "record", + "namespace" : "test", + "name" : "FirstLevel", + "fields" : [ + { "name" : "id1" , "type" : "int" }, + { "name" : "level1a" , "type" : "test.SecondLevelA" }, + { "name" : "level1b" , "type" : "test.SecondLevelB" } + ] + }`, + references: [ + { + name: 'test.SecondLevelA', + subject: 'Avro:SecondLevelA', + version: undefined, + }, + { + name: 'test.SecondLevelB', + subject: 'Avro:SecondLevelB', + version: undefined, + }, + ], + } as AvroConfluentSchema, + + SecondLevelASchema: { + type: SchemaType.AVRO, + schema: ` + { + "type" : "record", + "namespace" : "test", + "name" : "SecondLevelA", + "fields" : [ + { "name" : "id2a" , "type" : "int" }, + { "name" : "level2a" , "type" : "test.ThirdLevel" } + ] + }`, + references: [ + { + name: 'test.ThirdLevel', + subject: 'Avro:ThirdLevel', + version: undefined, + }, + ], + } as AvroConfluentSchema, + + SecondLevelBSchema: { + type: SchemaType.AVRO, + schema: ` + { + "type" : "record", + "namespace" : "test", + "name" : "SecondLevelB", + "fields" : [ + { "name" : "id2b" , "type" : "int" }, + { "name" : "level2b" , "type" : "test.ThirdLevel" } + ] + }`, + references: [ + { + name: 'test.ThirdLevel', + subject: 'Avro:ThirdLevel', + version: undefined, + }, + ], + } as AvroConfluentSchema, + + ThirdLevelSchema: { + type: SchemaType.AVRO, + schema: ` + { + "type" : "record", + "namespace" : "test", + "name" : "ThirdLevel", + "fields" : [ + { "name" : "id3" , "type" : "int" } + ] + }`, + } as AvroConfluentSchema, +} + +function apiResponse(result) { + return JSON.parse(result.responseData) +} + +describe('SchemaRegistry', () => { + let schemaRegistry: SchemaRegistry + let registeredSchema: RegisteredSchema + let api + + beforeEach(async () => { + api = API(schemaRegistryAPIClientArgs) + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + }) + + describe('when register', () => { + describe('when no reference', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'Avro:ThirdLevel', + }) + }) + + it('should return schema id', async () => { + expect(registeredSchema.id).toEqual(expect.any(Number)) + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with reference', () => { + let schemaId + let referenceSchema + + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'Avro:ThirdLevel', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'Avro:SecondLevelA', + }) + schemaId = registeredSchema.id + + const schemaRaw = apiResponse(await api.Schema.find({ id: schemaId })) + referenceSchema = schemaRaw.references[0].subject + }) + + it('should return schema id', async () => { + expect(schemaId).toEqual(expect.any(Number)) + }) + + it('should create a schema with reference', async () => { + expect(referenceSchema).toEqual('Avro:ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multiple reference', () => { + beforeEach(async () => { + let latest + + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'Avro:ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'Avro:SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'Avro:SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'Avro:FirstLevel', + }) + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + + it('should be able to encode/decode independent', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) + + describe('_getSchema', () => { + let schema + + describe('no references', () => { + beforeEach(async () => { + registeredSchema = await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'Avro:ThirdLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match name', async () => { + expect(schema.name).toEqual('test.ThirdLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { id3: 3 } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with references', () => { + beforeEach(async () => { + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { subject: 'Avro:ThirdLevel' }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'Avro:SecondLevelA', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match name', async () => { + expect(schema.name).toEqual('test.SecondLevelA') + }) + + it('should be able to encode/decode', async () => { + const obj = { id2a: 2, level2a: { id3: 3 } } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + + describe('with multi references', () => { + beforeEach(async () => { + let latest + + await schemaRegistry.register(TestSchemas.ThirdLevelSchema, { + subject: 'Avro:ThirdLevel', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:ThirdLevel' })) + TestSchemas.SecondLevelASchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelASchema, { + subject: 'Avro:SecondLevelA', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:ThirdLevel' })) + TestSchemas.SecondLevelBSchema.references[0].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.SecondLevelBSchema, { + subject: 'Avro:SecondLevelB', + }) + + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:SecondLevelA' })) + TestSchemas.FirstLevelSchema.references[0].version = latest.version + latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:SecondLevelB' })) + TestSchemas.FirstLevelSchema.references[1].version = latest.version + registeredSchema = await schemaRegistry.register(TestSchemas.FirstLevelSchema, { + subject: 'Avro:FirstLevel', + }) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should return schema that match name', async () => { + expect(schema.name).toEqual('test.FirstLevel') + }) + + it('should be able to encode/decode', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) + + describe('when document example', () => { + it('should encode/decode', async () => { + const schemaA = ` + { + "type" : "record", + "namespace" : "test", + "name" : "A", + "fields" : [ + { "name" : "id" , "type" : "int" }, + { "name" : "b" , "type" : "test.B" } + ] + }` + + const schemaB = ` + { + "type" : "record", + "namespace" : "test", + "name" : "B", + "fields" : [ + { "name" : "id" , "type" : "int" } + ] + }` + + await schemaRegistry.register( + { type: SchemaType.AVRO, schema: schemaB }, + { subject: 'Avro:B' }, + ) + + const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:B' })) + + const { id } = await schemaRegistry.register( + { + type: SchemaType.AVRO, + schema: schemaA, + references: [ + { + name: 'test.B', + subject: 'Avro:B', + version, + }, + ], + }, + { subject: 'Avro:A' }, + ) + + const obj = { id: 1, b: { id: 2 } } + + const buffer = await schemaRegistry.encode(id, obj) + const decodedObj = await schemaRegistry.decode(buffer) + + expect(decodedObj).toEqual(obj) + }) + }) +}) diff --git a/src/SchemaRegistry.json.spec.ts b/src/SchemaRegistry.json.spec.ts index 1b3bd9d..a0fa7d7 100644 --- a/src/SchemaRegistry.json.spec.ts +++ b/src/SchemaRegistry.json.spec.ts @@ -204,6 +204,22 @@ describe('SchemaRegistry', () => { expect(resultObj).toEqual(obj) }) + + it('should be able to encode/decode independent', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) }) }) @@ -295,7 +311,7 @@ describe('SchemaRegistry', () => { }) }) - describe('when document example', async () => { + describe('when document example', () => { it('should encode/decode', async () => { const schemaA = ` { @@ -321,6 +337,8 @@ describe('SchemaRegistry', () => { { subject: 'JSON:B' }, ) + const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:B' })) + const { id } = await schemaRegistry.register( { type: SchemaType.JSON, @@ -329,7 +347,7 @@ describe('SchemaRegistry', () => { { name: 'https://sumup.com/schemas/B', subject: 'JSON:B', - version: 1, + version, }, ], }, diff --git a/src/SchemaRegistry.protobuf.spec.ts b/src/SchemaRegistry.protobuf.spec.ts index 2b3d80b..8fbdf10 100644 --- a/src/SchemaRegistry.protobuf.spec.ts +++ b/src/SchemaRegistry.protobuf.spec.ts @@ -199,6 +199,22 @@ describe('SchemaRegistry', () => { expect(resultObj).toEqual(obj) }) + + it('should be able to encode/decode independent', async () => { + const obj = { + id1: 1, + level1a: { id2a: 2, level2a: { id3: 3 } }, + level1b: { id2b: 4, level2b: { id3: 5 } }, + } + + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + const buffer = await schemaRegistry.encode(registeredSchema.id, obj) + + schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + const resultObj = await schemaRegistry.decode(buffer) + + expect(resultObj).toEqual(obj) + }) }) }) @@ -302,7 +318,7 @@ describe('SchemaRegistry', () => { }) }) - describe('when document example', async () => { + describe('when document example', () => { it('should encode/decode', async () => { const schemaA = ` syntax = "proto3"; @@ -327,6 +343,8 @@ describe('SchemaRegistry', () => { { subject: 'Proto:B' }, ) + const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:B' })) + const { id } = await schemaRegistry.register( { type: SchemaType.PROTOBUF, @@ -335,7 +353,7 @@ describe('SchemaRegistry', () => { { name: 'test/B.proto', subject: 'Proto:B', - version: 1, + version, }, ], }, diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 866bb39..5281944 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -22,7 +22,6 @@ import { SchemaRegistryAPIClientOptions, AvroConfluentSchema, SchemaResponse, - ProtoConfluentSchema, ProtocolOptions, SchemaHelper, ReferenceType, @@ -115,8 +114,8 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(confluentSchema.type) - this.options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) - const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) + const options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) + const schemaInstance = schemaFromConfluentSchema(confluentSchema, options) helper.validate(schemaInstance) let subject: ConfluentSubject @@ -147,17 +146,12 @@ export default class SchemaRegistry { } } - const references = - confluentSchema.type === SchemaType.PROTOBUF || confluentSchema.type === SchemaType.JSON - ? (confluentSchema as ProtoConfluentSchema).references - : undefined - const response = await this.api.Subject.register({ subject: subject.name, body: { schemaType: confluentSchema.type === SchemaType.AVRO ? undefined : confluentSchema.type, schema: confluentSchema.schema, - references, + references: confluentSchema.references, }, }) @@ -190,12 +184,14 @@ export default class SchemaRegistry { helper: SchemaHelper, referencesSet: Set, ): Promise<(string | RawAvroSchema)[]> { - const references = helper.getReferences(schema) || [] - // execute in parallel - const schemaPromise = references.map(reference => - this.getReferredSchemasFromReference(reference, helper, referencesSet), - ) - return (await Promise.all(schemaPromise)).flat() + const references = schema.references || [] + + let referredSchemas: (string | RawAvroSchema)[] = [] + for (const reference of references) { + const schemas = await this.getReferredSchemasFromReference(reference, helper, referencesSet) + referredSchemas = referredSchemas.concat(schemas.flat()) + } + return referredSchemas } async getReferredSchemasFromReference( @@ -239,8 +235,8 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(schemaType) const confluentSchema = helper.toConfluentSchema(foundSchema) - this.options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) - const schemaInstance = schemaFromConfluentSchema(confluentSchema, this.options) + const options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) + const schemaInstance = schemaFromConfluentSchema(confluentSchema, options) return this.cache.setSchema(registryId, schemaType, schemaInstance) } From 50cb210860d8edad6dd5d8022f66020412458c5c Mon Sep 17 00:00:00 2001 From: bifrost Date: Tue, 15 Mar 2022 15:03:53 +0100 Subject: [PATCH 16/25] improve-doc --- docs/schema-avro.md | 86 +++++++++++++++++++++++++++++++++++++++++++++ docs/schema-json.md | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100755 docs/schema-avro.md diff --git a/docs/schema-avro.md b/docs/schema-avro.md new file mode 100755 index 0000000..26cd654 --- /dev/null +++ b/docs/schema-avro.md @@ -0,0 +1,86 @@ +--- +id: schema-avro +title: Example Avro Schemas +sidebar_label: Example Avro Schemas +--- + +## Schema with references to other schemas + +You might want to split the Avro definition into several schemas one for each type. + +```json +{ + "type" : "record", + "namespace" : "test", + "name" : "A", + "fields" : [ + { "name" : "id" , "type" : "int" }, + { "name" : "b" , "type" : "test.B" } + ] +} +``` + +```json +{ + "type" : "record", + "namespace" : "test", + "name" : "B", + "fields" : [ + { "name" : "id" , "type" : "int" } + ] +} +``` + +To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the namespace + name, a schema `subject` and a schema `version`. + +Notice the library will handle an arbitrary number of nested levels. + +```js +const schemaA = ` +{ + "type" : "record", + "namespace" : "test", + "name" : "A", + "fields" : [ + { "name" : "id" , "type" : "int" }, + { "name" : "b" , "type" : "test.B" } + ] +}` + +const schemaB = ` +{ + "type" : "record", + "namespace" : "test", + "name" : "B", + "fields" : [ + { "name" : "id" , "type" : "int" } + ] +}` + +await schemaRegistry.register( +{ type: SchemaType.AVRO, schema: schemaB }, +{ subject: 'B' }, +) + +const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'B' })) + +const { id } = await schemaRegistry.register( +{ + type: SchemaType.AVRO, + schema: schemaA, + references: [ + { + name: 'test.B', + subject: 'B', + version, + }, + ], +}, +{ subject: 'A' }, +) + +const obj = { id: 1, b: { id: 2 } } + +const buffer = await schemaRegistry.encode(id, obj) +const decodedObj = await schemaRegistry.decode(buffer) +``` diff --git a/docs/schema-json.md b/docs/schema-json.md index b3b8164..5fa16aa 100755 --- a/docs/schema-json.md +++ b/docs/schema-json.md @@ -29,7 +29,7 @@ You might want to split the JSON definition into several schemas one for each ty } ``` -To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the import statement, a schema `subject` and a schema `version`. +To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the $ref, a schema `subject` and a schema `version`. Notice the library will handle an arbitrary number of nested levels. From 254577e4aa88125a5e1dec029ee6a6550fce0f54 Mon Sep 17 00:00:00 2001 From: bifrost Date: Tue, 15 Mar 2022 15:11:30 +0100 Subject: [PATCH 17/25] cleanup-ts --- src/AvroHelper.ts | 2 -- src/JsonHelper.ts | 2 -- src/ProtoHelper.ts | 2 -- 3 files changed, 6 deletions(-) diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index 72bb0d8..dd2d3b0 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -5,8 +5,6 @@ import { ConfluentSchema, SchemaHelper, ConfluentSubject, - ReferenceType, - AvroConfluentSchema, ProtocolOptions, } from './@types' import { ConfluentSchemaRegistryArgumentError } from './errors' diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index 7493ad0..b49b720 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -5,8 +5,6 @@ import { ConfluentSchema, SchemaResponse, SchemaType, - ReferenceType, - JsonConfluentSchema, ProtocolOptions, } from './@types' import { ConfluentSchemaRegistryError } from './errors' diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index 3c52a51..b2ce01f 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -5,8 +5,6 @@ import { ConfluentSchema, SchemaResponse, SchemaType, - ReferenceType, - ProtoConfluentSchema, ProtocolOptions, } from './@types' import { ConfluentSchemaRegistryError } from './errors' From 183be7f6d55aeee1c75fcf575c5576b457cff815 Mon Sep 17 00:00:00 2001 From: bifrost Date: Wed, 16 Mar 2022 08:41:45 +0100 Subject: [PATCH 18/25] refactor-referredSchemas --- src/@types.ts | 8 ++++---- src/AvroHelper.ts | 13 +++++-------- src/JsonHelper.ts | 8 ++++---- src/JsonSchema.ts | 7 ++++--- src/ProtoHelper.ts | 8 ++++---- src/ProtoSchema.ts | 5 +++-- src/SchemaRegistry.ts | 12 ++++++------ 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/@types.ts b/src/@types.ts index ef4b297..22ae4f5 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -14,20 +14,20 @@ export interface SchemaHelper { toConfluentSchema(data: SchemaResponse): ConfluentSchema updateOptionsFromSchemaReferences( options: ProtocolOptions, - referredSchemas: (string | RawAvroSchema)[], + referredSchemas: ConfluentSchema[], ): ProtocolOptions } export type AvroOptions = Partial & { - referredSchemas?: (string | RawAvroSchema)[] + referredSchemas?: AvroConfluentSchema[] } export type JsonOptions = ConstructorParameters[0] & { ajvInstance?: { compile: (schema: any) => ValidateFunction } - referredSchemas?: string[] + referredSchemas?: JsonConfluentSchema[] } -export type ProtoOptions = { messageName?: string; referredSchemas?: string[] } +export type ProtoOptions = { messageName?: string; referredSchemas?: ProtoConfluentSchema[] } export interface LegacyOptions { forSchemaOptions?: AvroOptions diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index dd2d3b0..062a1be 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -6,6 +6,7 @@ import { SchemaHelper, ConfluentSubject, ProtocolOptions, + AvroConfluentSchema, } from './@types' import { ConfluentSchemaRegistryArgumentError } from './errors' import avro, { ForSchemaOptions } from 'avsc' @@ -34,11 +35,7 @@ export default class AvroHelper implements SchemaHelper { function(_schema: avro.Schema, opts: ForSchemaOptions) { const avroOpts = opts as AvroOptions avroOpts?.referredSchemas?.forEach(subSchema => { - const confluentSchema = { - schema: subSchema, - type: SchemaType.AVRO, - } as ConfluentSchema - const rawSubSchema = me.getRawAvroSchema(confluentSchema) + const rawSubSchema = me.getRawAvroSchema(subSchema) avroOpts.typeHook = undefined avro.Type.forSchema(rawSubSchema, avroOpts) }) @@ -55,7 +52,7 @@ export default class AvroHelper implements SchemaHelper { } public getSubject( - schema: ConfluentSchema, + schema: AvroConfluentSchema, // @ts-ignore avroSchema: AvroSchema, separator: string, @@ -77,13 +74,13 @@ export default class AvroHelper implements SchemaHelper { return asRawAvroSchema.name != null && asRawAvroSchema.type != null } - public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + public toConfluentSchema(data: SchemaResponse): AvroConfluentSchema { return { type: SchemaType.AVRO, schema: data.schema, references: data.references } } updateOptionsFromSchemaReferences( options: ProtocolOptions, - referredSchemas: (string | RawAvroSchema)[], + referredSchemas: AvroConfluentSchema[], ): ProtocolOptions { const opts = options ?? {} return { ...opts, [SchemaType.AVRO]: { ...opts[SchemaType.AVRO], referredSchemas } } diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index b49b720..bf9d967 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -2,10 +2,10 @@ import { Schema, SchemaHelper, ConfluentSubject, - ConfluentSchema, SchemaResponse, SchemaType, ProtocolOptions, + JsonConfluentSchema, } from './@types' import { ConfluentSchemaRegistryError } from './errors' @@ -15,20 +15,20 @@ export default class JsonHelper implements SchemaHelper { } public getSubject( - _confluentSchema: ConfluentSchema, + _confluentSchema: JsonConfluentSchema, _schema: Schema, _separator: string, ): ConfluentSubject { throw new ConfluentSchemaRegistryError('not implemented yet') } - public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + public toConfluentSchema(data: SchemaResponse): JsonConfluentSchema { return { type: SchemaType.JSON, schema: data.schema, references: data.references } } updateOptionsFromSchemaReferences( options: ProtocolOptions, - referredSchemas: string[], + referredSchemas: JsonConfluentSchema[], ): ProtocolOptions { const opts = options ?? {} return { ...opts, [SchemaType.JSON]: { ...opts[SchemaType.JSON], referredSchemas } } diff --git a/src/JsonSchema.ts b/src/JsonSchema.ts index bfdc151..99ddeb7 100644 --- a/src/JsonSchema.ts +++ b/src/JsonSchema.ts @@ -29,9 +29,10 @@ export default class JsonSchema implements Schema { private getJsonSchema(schema: JsonConfluentSchema, opts?: JsonOptions) { const ajv = opts?.ajvInstance ?? new Ajv(opts) - if (opts?.referredSchemas) { - opts.referredSchemas.forEach(rawSchema => { - const $schema = JSON.parse(rawSchema) + const referredSchemas = opts?.referredSchemas + if (referredSchemas) { + referredSchemas.forEach(rawSchema => { + const $schema = JSON.parse(rawSchema.schema as string) // @ts-ignore ajv.addSchema($schema, $schema['$id']) }) diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index b2ce01f..6b9fc41 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -2,10 +2,10 @@ import { Schema, SchemaHelper, ConfluentSubject, - ConfluentSchema, SchemaResponse, SchemaType, ProtocolOptions, + ProtoConfluentSchema, } from './@types' import { ConfluentSchemaRegistryError } from './errors' @@ -15,20 +15,20 @@ export default class ProtoHelper implements SchemaHelper { } public getSubject( - _confluentSchema: ConfluentSchema, + _confluentSchema: ProtoConfluentSchema, _schema: Schema, _separator: string, ): ConfluentSubject { throw new ConfluentSchemaRegistryError('not implemented yet') } - public toConfluentSchema(data: SchemaResponse): ConfluentSchema { + public toConfluentSchema(data: SchemaResponse): ProtoConfluentSchema { return { type: SchemaType.PROTOBUF, schema: data.schema, references: data.references } } updateOptionsFromSchemaReferences( options: ProtocolOptions, - referredSchemas: string[], + referredSchemas: ProtoConfluentSchema[], ): ProtocolOptions { const opts = options ?? {} return { ...opts, [SchemaType.PROTOBUF]: { ...opts[SchemaType.PROTOBUF], referredSchemas } } diff --git a/src/ProtoSchema.ts b/src/ProtoSchema.ts index 350d4b4..282e07a 100644 --- a/src/ProtoSchema.ts +++ b/src/ProtoSchema.ts @@ -12,10 +12,11 @@ export default class ProtoSchema implements Schema { constructor(schema: ProtoConfluentSchema, opts?: ProtoOptions) { const parsedMessage = protobuf.parse(schema.schema) const root = parsedMessage.root + const referredSchemas = opts?.referredSchemas // handle all schema references independent on nested references - if (opts?.referredSchemas) { - opts.referredSchemas.forEach(rawSchema => protobuf.parse(rawSchema, root)) + if (referredSchemas) { + referredSchemas.forEach(rawSchema => protobuf.parse(rawSchema.schema as string, root)) } this.message = root.lookupType(this.getTypeName(parsedMessage, opts)) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 5281944..046bed1 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -174,7 +174,7 @@ export default class SchemaRegistry { private async getReferredSchemas( schema: ConfluentSchema, helper: SchemaHelper, - ): Promise<(string | RawAvroSchema)[]> { + ): Promise { const referencesSet = new Set() return this.getReferredSchemasRecursive(schema, helper, referencesSet) } @@ -183,13 +183,13 @@ export default class SchemaRegistry { schema: ConfluentSchema, helper: SchemaHelper, referencesSet: Set, - ): Promise<(string | RawAvroSchema)[]> { + ): Promise { const references = schema.references || [] - let referredSchemas: (string | RawAvroSchema)[] = [] + let referredSchemas: ConfluentSchema[] = [] for (const reference of references) { const schemas = await this.getReferredSchemasFromReference(reference, helper, referencesSet) - referredSchemas = referredSchemas.concat(schemas.flat()) + referredSchemas = referredSchemas.concat(schemas) } return referredSchemas } @@ -198,7 +198,7 @@ export default class SchemaRegistry { reference: ReferenceType, helper: SchemaHelper, referencesSet: Set, - ): Promise<(string | RawAvroSchema)[]> { + ): Promise { const { name, subject, version } = reference const key = `${name}-${subject}-${version}` @@ -214,7 +214,7 @@ export default class SchemaRegistry { const schema = helper.toConfluentSchema(foundSchema) const referredSchemas = await this.getReferredSchemasRecursive(schema, helper, referencesSet) - referredSchemas.push(schema.schema) + referredSchemas.push(schema) return referredSchemas } From 90b41e044ab2e8b98d1b49568739a2d5dee17e9b Mon Sep 17 00:00:00 2001 From: bifrost Date: Thu, 17 Mar 2022 12:33:27 +0100 Subject: [PATCH 19/25] efactoring-and-improve-examples --- docs/schema-avro.md | 47 +++++++++++----------- docs/schema-json.md | 61 ++++++++++++++++------------- docs/schema-protobuf.md | 27 ++++++++----- src/AvroHelper.ts | 21 +++++----- src/JsonSchema.ts | 2 +- src/SchemaRegistry.avro.spec.ts | 43 ++++++++++---------- src/SchemaRegistry.json.spec.ts | 41 +++++++++---------- src/SchemaRegistry.protobuf.spec.ts | 3 +- 8 files changed, 125 insertions(+), 120 deletions(-) diff --git a/docs/schema-avro.md b/docs/schema-avro.md index 26cd654..49d2f4d 100755 --- a/docs/schema-avro.md +++ b/docs/schema-avro.md @@ -36,47 +36,44 @@ To registry schemas with references they have to be registered in reverse order, Notice the library will handle an arbitrary number of nested levels. ```js -const schemaA = ` -{ - "type" : "record", - "namespace" : "test", - "name" : "A", - "fields" : [ - { "name" : "id" , "type" : "int" }, - { "name" : "b" , "type" : "test.B" } - ] -}` +const schemaA = { + type: 'record', + namespace: 'test', + name: 'A', + fields: [ + { name: 'id', type: 'int' }, + { name: 'b', type: 'test.B' }, + ], +} -const schemaB = ` -{ - "type" : "record", - "namespace" : "test", - "name" : "B", - "fields" : [ - { "name" : "id" , "type" : "int" } - ] -}` +const schemaB = { + type: 'record', + namespace: 'test', + name: 'B', + fields: [{ name: 'id', type: 'int' }], +} await schemaRegistry.register( -{ type: SchemaType.AVRO, schema: schemaB }, -{ subject: 'B' }, + { type: SchemaType.AVRO, schema: JSON.stringify(schemaB) }, + { subject: 'Avro:B' }, ) -const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'B' })) +const response = await schemaRegistry.api.Subject.latestVersion({ subject: 'Avro:B' }) +const { version } = JSON.parse(response.responseData) const { id } = await schemaRegistry.register( { type: SchemaType.AVRO, - schema: schemaA, + schema: JSON.stringify(schemaA), references: [ { name: 'test.B', - subject: 'B', + subject: 'Avro:B', version, }, ], }, -{ subject: 'A' }, +{ subject: 'Avro:A' }, ) const obj = { id: 1, b: { id: 2 } } diff --git a/docs/schema-json.md b/docs/schema-json.md index 5fa16aa..3c3fb02 100755 --- a/docs/schema-json.md +++ b/docs/schema-json.md @@ -34,40 +34,45 @@ To registry schemas with references they have to be registered in reverse order, Notice the library will handle an arbitrary number of nested levels. ```js -const schemaA = ` -{ - "$id": "https://sumup.com/schemas/A", - "type": "object", - "properties": { - "id": { "type": "number" }, - "b": { "$ref": "https://sumup.com/schemas/B" } - } -}` +const schemaA = { + $id: 'https://sumup.com/schemas/A', + type: 'object', + properties: { + id: { type: 'number' }, + b: { $ref: 'https://sumup.com/schemas/B' }, + }, +} -const schemaB = ` -{ - "$id": "https://sumup.com/schemas/B", - "type": "object", - "properties": { - "id": { "type": "number" } - } -}` +const schemaB = { + $id: 'https://sumup.com/schemas/B', + type: 'object', + properties: { + id: { type: 'number' }, + }, +} await schemaRegistry.register( - { type: SchemaType.JSON, schema: schemaB }, - { subject: 'B'}) + { type: SchemaType.JSON, schema: JSON.stringify(schemaB) }, + { subject: 'JSON:B' }, +) -const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'B' })) +const response = await schemaRegistry.api.Subject.latestVersion({ subject: 'JSON:B' }) +const { version } = JSON.parse(response.responseData) const { id } = await schemaRegistry.register( - { type: SchemaType.JSON, schema: schemaA, references: [ - { - name: 'https://sumup.com/schemas/B', - subject: 'B', - version, - }, - ]}, - { subject: 'A' }) +{ + type: SchemaType.JSON, + schema: JSON.stringify(schemaA), + references: [ + { + name: 'https://sumup.com/schemas/B', + subject: 'JSON:B', + version, + }, + ], +}, +{ subject: 'JSON:A' }, +) const obj = { id: 1, b: { id: 2 } } diff --git a/docs/schema-protobuf.md b/docs/schema-protobuf.md index 9f74c03..8c50356 100755 --- a/docs/schema-protobuf.md +++ b/docs/schema-protobuf.md @@ -53,19 +53,26 @@ const schemaB = ` await schemaRegistry.register( { type: SchemaType.PROTOBUF, schema: schemaB }, - { subject: 'B'}) + { subject: 'Proto:B' }, +) -const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'B' })) +const response = await schemaRegistry.api.Subject.latestVersion({ subject: 'Proto:B' }) +const { version } = JSON.parse(response.responseData) const { id } = await schemaRegistry.register( - { type: SchemaType.PROTOBUF, schema: schemaA, references: [ - { - name: 'test/B.proto', - subject: 'B', - version, - }, - ]}, - { subject: 'A' }) +{ + type: SchemaType.PROTOBUF, + schema: schemaA, + references: [ + { + name: 'test/B.proto', + subject: 'Proto:B', + version, + }, + ], +}, +{ subject: 'Proto:A' }, +) const obj = { id: 1, b: { id: 2 } } diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index 062a1be..90691f0 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -25,21 +25,18 @@ export default class AvroHelper implements SchemaHelper { : this.getRawAvroSchema(schema) // @ts-ignore TODO: Fix typings for Schema... - // eslint-disable-next-line @typescript-eslint/no-this-alias - const me = this + const typeHook = (_schema: avro.Schema, opts: ForSchemaOptions) => { + const avroOpts = opts as AvroOptions + avroOpts?.referredSchemas?.forEach(subSchema => { + const rawSubSchema = this.getRawAvroSchema(subSchema) + avroOpts.typeHook = undefined + avro.Type.forSchema(rawSubSchema, avroOpts) + }) + } const avroSchema = avro.Type.forSchema(rawSchema, { ...opts, // @ts-ignore - typeHook: - opts?.typeHook || - function(_schema: avro.Schema, opts: ForSchemaOptions) { - const avroOpts = opts as AvroOptions - avroOpts?.referredSchemas?.forEach(subSchema => { - const rawSubSchema = me.getRawAvroSchema(subSchema) - avroOpts.typeHook = undefined - avro.Type.forSchema(rawSubSchema, avroOpts) - }) - }, + typeHook: opts?.typeHook || typeHook, }) return avroSchema diff --git a/src/JsonSchema.ts b/src/JsonSchema.ts index 99ddeb7..ce72a13 100644 --- a/src/JsonSchema.ts +++ b/src/JsonSchema.ts @@ -32,7 +32,7 @@ export default class JsonSchema implements Schema { const referredSchemas = opts?.referredSchemas if (referredSchemas) { referredSchemas.forEach(rawSchema => { - const $schema = JSON.parse(rawSchema.schema as string) + const $schema = JSON.parse(rawSchema.schema) // @ts-ignore ajv.addSchema($schema, $schema['$id']) }) diff --git a/src/SchemaRegistry.avro.spec.ts b/src/SchemaRegistry.avro.spec.ts index 64812b3..fe03f5e 100644 --- a/src/SchemaRegistry.avro.spec.ts +++ b/src/SchemaRegistry.avro.spec.ts @@ -326,38 +326,35 @@ describe('SchemaRegistry', () => { describe('when document example', () => { it('should encode/decode', async () => { - const schemaA = ` - { - "type" : "record", - "namespace" : "test", - "name" : "A", - "fields" : [ - { "name" : "id" , "type" : "int" }, - { "name" : "b" , "type" : "test.B" } - ] - }` - - const schemaB = ` - { - "type" : "record", - "namespace" : "test", - "name" : "B", - "fields" : [ - { "name" : "id" , "type" : "int" } - ] - }` + const schemaA = { + type: 'record', + namespace: 'test', + name: 'A', + fields: [ + { name: 'id', type: 'int' }, + { name: 'b', type: 'test.B' }, + ], + } + + const schemaB = { + type: 'record', + namespace: 'test', + name: 'B', + fields: [{ name: 'id', type: 'int' }], + } await schemaRegistry.register( - { type: SchemaType.AVRO, schema: schemaB }, + { type: SchemaType.AVRO, schema: JSON.stringify(schemaB) }, { subject: 'Avro:B' }, ) - const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:B' })) + const response = await schemaRegistry.api.Subject.latestVersion({ subject: 'Avro:B' }) + const { version } = JSON.parse(response.responseData) const { id } = await schemaRegistry.register( { type: SchemaType.AVRO, - schema: schemaA, + schema: JSON.stringify(schemaA), references: [ { name: 'test.B', diff --git a/src/SchemaRegistry.json.spec.ts b/src/SchemaRegistry.json.spec.ts index a0fa7d7..b2114be 100644 --- a/src/SchemaRegistry.json.spec.ts +++ b/src/SchemaRegistry.json.spec.ts @@ -313,36 +313,35 @@ describe('SchemaRegistry', () => { describe('when document example', () => { it('should encode/decode', async () => { - const schemaA = ` - { - "$id": "https://sumup.com/schemas/A", - "type": "object", - "properties": { - "id": { "type": "number" }, - "b": { "$ref": "https://sumup.com/schemas/B" } - } - }` + const schemaA = { + $id: 'https://sumup.com/schemas/A', + type: 'object', + properties: { + id: { type: 'number' }, + b: { $ref: 'https://sumup.com/schemas/B' }, + }, + } - const schemaB = ` - { - "$id": "https://sumup.com/schemas/B", - "type": "object", - "properties": { - "id": { "type": "number" } - } - }` + const schemaB = { + $id: 'https://sumup.com/schemas/B', + type: 'object', + properties: { + id: { type: 'number' }, + }, + } await schemaRegistry.register( - { type: SchemaType.JSON, schema: schemaB }, + { type: SchemaType.JSON, schema: JSON.stringify(schemaB) }, { subject: 'JSON:B' }, ) - const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'JSON:B' })) + const response = await schemaRegistry.api.Subject.latestVersion({ subject: 'JSON:B' }) + const { version } = JSON.parse(response.responseData) const { id } = await schemaRegistry.register( { type: SchemaType.JSON, - schema: schemaA, + schema: JSON.stringify(schemaA), references: [ { name: 'https://sumup.com/schemas/B', @@ -358,6 +357,8 @@ describe('SchemaRegistry', () => { const buffer = await schemaRegistry.encode(id, obj) const decodedObj = await schemaRegistry.decode(buffer) + + expect(decodedObj).toEqual(obj) }) }) }) diff --git a/src/SchemaRegistry.protobuf.spec.ts b/src/SchemaRegistry.protobuf.spec.ts index 8fbdf10..7477a9f 100644 --- a/src/SchemaRegistry.protobuf.spec.ts +++ b/src/SchemaRegistry.protobuf.spec.ts @@ -343,7 +343,8 @@ describe('SchemaRegistry', () => { { subject: 'Proto:B' }, ) - const { version } = apiResponse(await api.Subject.latestVersion({ subject: 'Proto:B' })) + const response = await schemaRegistry.api.Subject.latestVersion({ subject: 'Proto:B' }) + const { version } = JSON.parse(response.responseData) const { id } = await schemaRegistry.register( { From a3db4e0cb0ec68b3cb58a7d628105bdb18ecb698 Mon Sep 17 00:00:00 2001 From: bifrost Date: Tue, 22 Mar 2022 13:35:27 +0100 Subject: [PATCH 20/25] enable-typeHook-for-referred-schemas --- src/AvroHelper.ts | 17 ++-- src/SchemaRegistry.avro.spec.ts | 148 ++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index 90691f0..cb10990 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -9,9 +9,10 @@ import { AvroConfluentSchema, } from './@types' import { ConfluentSchemaRegistryArgumentError } from './errors' -import avro, { ForSchemaOptions } from 'avsc' +import avro, { ForSchemaOptions, Schema, Type } from 'avsc' import { SchemaResponse, SchemaType } from './@types' +type TypeHook = (schema: Schema, opts: ForSchemaOptions) => Type export default class AvroHelper implements SchemaHelper { private getRawAvroSchema(schema: ConfluentSchema): RawAvroSchema { return (typeof schema.schema === 'string' @@ -25,18 +26,24 @@ export default class AvroHelper implements SchemaHelper { : this.getRawAvroSchema(schema) // @ts-ignore TODO: Fix typings for Schema... - const typeHook = (_schema: avro.Schema, opts: ForSchemaOptions) => { + const addReferredSchemas = (userHook?: TypeHook): TypeHook => ( + schema: avro.Schema, + opts: ForSchemaOptions, + ) => { const avroOpts = opts as AvroOptions avroOpts?.referredSchemas?.forEach(subSchema => { const rawSubSchema = this.getRawAvroSchema(subSchema) - avroOpts.typeHook = undefined + avroOpts.typeHook = userHook avro.Type.forSchema(rawSubSchema, avroOpts) }) + if (userHook) { + return userHook(schema, opts) + } } + const avroSchema = avro.Type.forSchema(rawSchema, { ...opts, - // @ts-ignore - typeHook: opts?.typeHook || typeHook, + typeHook: addReferredSchemas(opts?.typeHook), }) return avroSchema diff --git a/src/SchemaRegistry.avro.spec.ts b/src/SchemaRegistry.avro.spec.ts index fe03f5e..8cf3919 100644 --- a/src/SchemaRegistry.avro.spec.ts +++ b/src/SchemaRegistry.avro.spec.ts @@ -1,11 +1,23 @@ import SchemaRegistry, { RegisteredSchema } from './SchemaRegistry' import API from './api' import { AvroConfluentSchema, SchemaType } from './@types' +import avro from 'avsc' const REGISTRY_HOST = 'http://localhost:8982' const schemaRegistryAPIClientArgs = { host: REGISTRY_HOST } const schemaRegistryArgs = { host: REGISTRY_HOST } +enum Color { + RED = 1, + GREEN = 2, + BLUE = 3, +} + +enum Direction { + UP = 1, + DOWN = 2, +} + const TestSchemas = { FirstLevelSchema: { type: SchemaType.AVRO, @@ -88,6 +100,56 @@ const TestSchemas = { ] }`, } as AvroConfluentSchema, + + EnumSchema: { + type: SchemaType.AVRO, + schema: ` + { + "type" : "record", + "namespace" : "test", + "name" : "EnumSchema", + "fields" : [ + { + "name": "color", + "type": ["null", { + "type": "enum", + "name": "Color", + "symbols": ["RED", "GREEN", "BLUE"] + } + ] + } + ] + }`, + } as AvroConfluentSchema, + + EnumWithReferencesSchema: { + type: SchemaType.AVRO, + schema: ` + { + "type" : "record", + "namespace" : "test", + "name" : "EnumWithReferences", + "fields" : [ + { + "name": "direction", + "type": ["null", { + "type": "enum", + "name": "Direction", + "symbols": ["UP", "DOWN"] + } + ] + }, + { "name" : "attributes" , "type" : "test.EnumSchema" } + ] + }`, + references: [ + { + name: 'test.EnumSchema', + subject: 'Avro:EnumSchema', + version: undefined, + }, + ], + } as AvroConfluentSchema, } function apiResponse(result) { @@ -374,4 +436,90 @@ describe('SchemaRegistry', () => { expect(decodedObj).toEqual(obj) }) }) + + describe('with EnumType types and nested schemas', () => { + /** + * Hook which will decode/encode enums to/from integers. + * + * The default `EnumType` implementation represents enum values as strings + * (consistent with the JSON representation). This hook can be used to provide + * an alternate representation (which is for example compatible with TypeScript + * enums). + * + * For simplicity, we don't do any bound checking here but we could by + * implementing a "bounded long" logical type and returning that instead. + * + * https://gist.github.com/mtth/c0088c745de048c4e466#file-long-enum-js + */ + function typeHook(attrs, opts) { + if (attrs.type === 'enum') { + return avro.parse('long', opts) + } + } + + let schema + + describe('with no enum typeHook defined', () => { + beforeEach(async () => { + const schemaRegistry = new SchemaRegistry(schemaRegistryArgs) + + await schemaRegistry.register(TestSchemas.EnumSchema, { + subject: 'Avro:EnumSchema', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:EnumSchema' })) + TestSchemas.EnumWithReferencesSchema.references[0].version = latest.version + const registeredSchema = await schemaRegistry.register( + TestSchemas.EnumWithReferencesSchema, + { + subject: 'Avro:EnumWithReferences', + }, + ) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should not be able to encode/decode enums schemas', async () => { + const obj = { + direction: Direction.UP, + attributes: { color: Color.BLUE }, + } + + expect(() => schema.toBuffer(obj)).toThrow(Error) + }) + }) + + describe('with enum typeHook defined', () => { + beforeEach(async () => { + const schemaRegistry = new SchemaRegistry(schemaRegistryArgs, { + [SchemaType.AVRO]: { typeHook }, + }) + + await schemaRegistry.register(TestSchemas.EnumSchema, { + subject: 'Avro:EnumSchema', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:EnumSchema' })) + TestSchemas.EnumWithReferencesSchema.references[0].version = latest.version + const registeredSchema = await schemaRegistry.register( + TestSchemas.EnumWithReferencesSchema, + { + subject: 'Avro:EnumWithReferences', + }, + ) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + + it('should be able to encode/decode enums schemas', async () => { + const obj = { + direction: Direction.UP, + attributes: { color: Color.BLUE }, + } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + }) }) From 3c5fefecc16676f6b83277338cbeb2effb20a742 Mon Sep 17 00:00:00 2001 From: bifrost Date: Fri, 30 Sep 2022 15:08:39 +0200 Subject: [PATCH 21/25] fix-review-comments-2 --- docs/schema-avro.md | 8 +++-- docs/schema-json.md | 22 ++++++++------ docs/schema-protobuf.md | 10 ++++-- package.json | 2 +- src/@types.ts | 20 ++++++------ src/AvroHelper.ts | 12 ++++---- src/JsonHelper.ts | 6 ++-- src/JsonSchema.ts | 6 ++-- src/ProtoHelper.ts | 6 ++-- src/ProtoSchema.ts | 6 ++-- src/SchemaRegistry.avro.spec.ts | 33 ++++++++++++++++++++ src/SchemaRegistry.json.spec.ts | 32 +++++++++---------- src/SchemaRegistry.ts | 54 +++++++++++++++++++++------------ 13 files changed, 139 insertions(+), 78 deletions(-) diff --git a/docs/schema-avro.md b/docs/schema-avro.md index 49d2f4d..cac2fd9 100755 --- a/docs/schema-avro.md +++ b/docs/schema-avro.md @@ -31,9 +31,13 @@ You might want to split the Avro definition into several schemas one for each ty } ``` -To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the namespace + name, a schema `subject` and a schema `version`. +To register schemas with references, the schemas have to be registered in reverse order. The schema that references another schema has to be registered after the schema it references. In this example B has to be registered before A. Furthermore, when registering A, a list of references have to be provided. A reference consist of: -Notice the library will handle an arbitrary number of nested levels. + * `name` - the fully qualified name of the referenced schema. Example: `test.B` + * `subject` - the subject the schema is registered under in the registry + * `version` - the version of the schema you want to use + +The library will handle an arbitrary number of nested levels. ```js const schemaA = { diff --git a/docs/schema-json.md b/docs/schema-json.md index 3c3fb02..82d3915 100755 --- a/docs/schema-json.md +++ b/docs/schema-json.md @@ -10,18 +10,18 @@ You might want to split the JSON definition into several schemas one for each ty ```JSON { - "$id": "https://sumup.com/schemas/A", + "$id": "https://example.com/schemas/A", "type": "object", "properties": { "id": { "type": "number" }, - "b": { "$ref": "https://sumup.com/schemas/B" } + "b": { "$ref": "https://example.com/schemas/B" } } } ``` ```JSON { - "$id": "https://sumup.com/schemas/B", + "$id": "https://example.com/schemas/B", "type": "object", "properties": { "id": { "type": "number" } @@ -29,22 +29,26 @@ You might want to split the JSON definition into several schemas one for each ty } ``` -To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the $ref, a schema `subject` and a schema `version`. +To register schemas with references, the schemas have to be registered in reverse order. The schema that references another schema has to be registered after the schema it references. In this example B has to be registered before A. Furthermore, when registering A, a list of references have to be provided. A reference consist of: -Notice the library will handle an arbitrary number of nested levels. + * `name` - A URL matching the `$ref` from the schema + * `subject` - the subject the schema is registered under in the registry + * `version` - the version of the schema you want to use + +The library will handle an arbitrary number of nested levels. ```js const schemaA = { - $id: 'https://sumup.com/schemas/A', + $id: 'https://example.com/schemas/A', type: 'object', properties: { id: { type: 'number' }, - b: { $ref: 'https://sumup.com/schemas/B' }, + b: { $ref: 'https://example.com/schemas/B' }, }, } const schemaB = { - $id: 'https://sumup.com/schemas/B', + $id: 'https://example.com/schemas/B', type: 'object', properties: { id: { type: 'number' }, @@ -65,7 +69,7 @@ const { id } = await schemaRegistry.register( schema: JSON.stringify(schemaA), references: [ { - name: 'https://sumup.com/schemas/B', + name: 'https://example.com/schemas/B', subject: 'JSON:B', version, }, diff --git a/docs/schema-protobuf.md b/docs/schema-protobuf.md index 8c50356..c2421ab 100755 --- a/docs/schema-protobuf.md +++ b/docs/schema-protobuf.md @@ -6,7 +6,7 @@ sidebar_label: Example Protobuf Schemas ## Schema with references to other schemas -You might want to split the Protobuf definition into several schemas one for each type. +You might want to split the Protobuf definition into several schemas, one for each type. ```protobuf syntax = "proto3"; @@ -28,9 +28,13 @@ message B { } ``` -To registry schemas with references they have to be registered in reverse order, so the referred schemas already exists. In this case B has to be registered before A. Furthermore A must define an array references to the referred schemas. A reference consist of a `name`, that should match the import statement, a schema `subject` and a schema `version`. +To register schemas with references, the schemas have to be registered in reverse order. The schema that references another schema has to be registered after the schema it references. In this example B has to be registered before A. Furthermore, when registering A, a list of references have to be provided. A reference consist of: -Notice the library will handle an arbitrary number of nested levels. + * `name` - String matching the import statement. For example: `test/B.proto` + * `subject` - the subject the schema is registered under in the registry + * `version` - the version of the schema you want to use + +The library will handle an arbitrary number of nested levels. ```js const schemaA = ` diff --git a/package.json b/package.json index ad8d74e..b45e0ef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@kafkajs/confluent-schema-registry", - "version": "3.3.0", + "version": "3.2.1", "main": "dist/index.js", "description": "ConfluentSchemaRegistry is a library that makes it easier to interact with the Confluent schema registry, it provides convenient methods to encode, decode and register new schemas using the Apache Avro serialization format.", "keywords": [ diff --git a/src/@types.ts b/src/@types.ts index 22ae4f5..e4a51af 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -13,21 +13,21 @@ export interface SchemaHelper { getSubject(confluentSchema: ConfluentSchema, schema: Schema, separator: string): ConfluentSubject toConfluentSchema(data: SchemaResponse): ConfluentSchema updateOptionsFromSchemaReferences( - options: ProtocolOptions, - referredSchemas: ConfluentSchema[], + referencedSchemas: ConfluentSchema[], + options?: ProtocolOptions, ): ProtocolOptions } export type AvroOptions = Partial & { - referredSchemas?: AvroConfluentSchema[] + referencedSchemas?: AvroConfluentSchema[] } export type JsonOptions = ConstructorParameters[0] & { ajvInstance?: { compile: (schema: any) => ValidateFunction } - referredSchemas?: JsonConfluentSchema[] + referencedSchemas?: JsonConfluentSchema[] } -export type ProtoOptions = { messageName?: string; referredSchemas?: ProtoConfluentSchema[] } +export type ProtoOptions = { messageName?: string; referencedSchemas?: ProtoConfluentSchema[] } export interface LegacyOptions { forSchemaOptions?: AvroOptions @@ -67,10 +67,10 @@ export interface ConfluentSubject { export interface AvroConfluentSchema { type: SchemaType.AVRO schema: string | RawAvroSchema - references?: ReferenceType[] + references?: SchemaReference[] } -export type ReferenceType = { +export type SchemaReference = { name: string subject: string version: number @@ -78,17 +78,17 @@ export type ReferenceType = { export interface ProtoConfluentSchema { type: SchemaType.PROTOBUF schema: string - references?: ReferenceType[] + references?: SchemaReference[] } export interface JsonConfluentSchema { type: SchemaType.JSON schema: string - references?: ReferenceType[] + references?: SchemaReference[] } export interface SchemaResponse { schema: string schemaType: string - references?: ReferenceType[] + references?: SchemaReference[] } export type ConfluentSchema = AvroConfluentSchema | ProtoConfluentSchema | JsonConfluentSchema diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index cb10990..d7e63f8 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -26,12 +26,12 @@ export default class AvroHelper implements SchemaHelper { : this.getRawAvroSchema(schema) // @ts-ignore TODO: Fix typings for Schema... - const addReferredSchemas = (userHook?: TypeHook): TypeHook => ( + const addReferencedSchemas = (userHook?: TypeHook): TypeHook => ( schema: avro.Schema, opts: ForSchemaOptions, ) => { const avroOpts = opts as AvroOptions - avroOpts?.referredSchemas?.forEach(subSchema => { + avroOpts?.referencedSchemas?.forEach(subSchema => { const rawSubSchema = this.getRawAvroSchema(subSchema) avroOpts.typeHook = userHook avro.Type.forSchema(rawSubSchema, avroOpts) @@ -43,7 +43,7 @@ export default class AvroHelper implements SchemaHelper { const avroSchema = avro.Type.forSchema(rawSchema, { ...opts, - typeHook: addReferredSchemas(opts?.typeHook), + typeHook: addReferencedSchemas(opts?.typeHook), }) return avroSchema @@ -83,10 +83,10 @@ export default class AvroHelper implements SchemaHelper { } updateOptionsFromSchemaReferences( - options: ProtocolOptions, - referredSchemas: AvroConfluentSchema[], + referencedSchemas: AvroConfluentSchema[], + options?: ProtocolOptions, ): ProtocolOptions { const opts = options ?? {} - return { ...opts, [SchemaType.AVRO]: { ...opts[SchemaType.AVRO], referredSchemas } } + return { ...opts, [SchemaType.AVRO]: { ...opts[SchemaType.AVRO], referencedSchemas } } } } diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index bf9d967..726c89b 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -27,10 +27,10 @@ export default class JsonHelper implements SchemaHelper { } updateOptionsFromSchemaReferences( - options: ProtocolOptions, - referredSchemas: JsonConfluentSchema[], + referencedSchemas: JsonConfluentSchema[], + options?: ProtocolOptions, ): ProtocolOptions { const opts = options ?? {} - return { ...opts, [SchemaType.JSON]: { ...opts[SchemaType.JSON], referredSchemas } } + return { ...opts, [SchemaType.JSON]: { ...opts[SchemaType.JSON], referencedSchemas } } } } diff --git a/src/JsonSchema.ts b/src/JsonSchema.ts index ce72a13..e103396 100644 --- a/src/JsonSchema.ts +++ b/src/JsonSchema.ts @@ -29,9 +29,9 @@ export default class JsonSchema implements Schema { private getJsonSchema(schema: JsonConfluentSchema, opts?: JsonOptions) { const ajv = opts?.ajvInstance ?? new Ajv(opts) - const referredSchemas = opts?.referredSchemas - if (referredSchemas) { - referredSchemas.forEach(rawSchema => { + const referencedSchemas = opts?.referencedSchemas + if (referencedSchemas) { + referencedSchemas.forEach(rawSchema => { const $schema = JSON.parse(rawSchema.schema) // @ts-ignore ajv.addSchema($schema, $schema['$id']) diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index 6b9fc41..9ffe8a6 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -27,10 +27,10 @@ export default class ProtoHelper implements SchemaHelper { } updateOptionsFromSchemaReferences( - options: ProtocolOptions, - referredSchemas: ProtoConfluentSchema[], + referencedSchemas: ProtoConfluentSchema[], + options?: ProtocolOptions, ): ProtocolOptions { const opts = options ?? {} - return { ...opts, [SchemaType.PROTOBUF]: { ...opts[SchemaType.PROTOBUF], referredSchemas } } + return { ...opts, [SchemaType.PROTOBUF]: { ...opts[SchemaType.PROTOBUF], referencedSchemas } } } } diff --git a/src/ProtoSchema.ts b/src/ProtoSchema.ts index 282e07a..bd358fe 100644 --- a/src/ProtoSchema.ts +++ b/src/ProtoSchema.ts @@ -12,11 +12,11 @@ export default class ProtoSchema implements Schema { constructor(schema: ProtoConfluentSchema, opts?: ProtoOptions) { const parsedMessage = protobuf.parse(schema.schema) const root = parsedMessage.root - const referredSchemas = opts?.referredSchemas + const referencedSchemas = opts?.referencedSchemas // handle all schema references independent on nested references - if (referredSchemas) { - referredSchemas.forEach(rawSchema => protobuf.parse(rawSchema.schema as string, root)) + if (referencedSchemas) { + referencedSchemas.forEach(rawSchema => protobuf.parse(rawSchema.schema as string, root)) } this.message = root.lookupType(this.getTypeName(parsedMessage, opts)) diff --git a/src/SchemaRegistry.avro.spec.ts b/src/SchemaRegistry.avro.spec.ts index 8cf3919..7bf92da 100644 --- a/src/SchemaRegistry.avro.spec.ts +++ b/src/SchemaRegistry.avro.spec.ts @@ -509,6 +509,39 @@ describe('SchemaRegistry', () => { ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) }) + it('should be able to encode/decode enums schemas', async () => { + const obj = { + direction: Direction.UP, + attributes: { color: Color.BLUE }, + } + + const buffer = await schema.toBuffer(obj) + const resultObj = await schema.fromBuffer(buffer) + + expect(resultObj).toEqual(obj) + }) + }) + describe('with enum typeHook defined as LegacyOptions', () => { + beforeEach(async () => { + const schemaRegistry = new SchemaRegistry(schemaRegistryArgs, { + forSchemaOptions: { typeHook }, + }) + + await schemaRegistry.register(TestSchemas.EnumSchema, { + subject: 'Avro:EnumSchema', + }) + + const latest = apiResponse(await api.Subject.latestVersion({ subject: 'Avro:EnumSchema' })) + TestSchemas.EnumWithReferencesSchema.references[0].version = latest.version + const registeredSchema = await schemaRegistry.register( + TestSchemas.EnumWithReferencesSchema, + { + subject: 'Avro:EnumWithReferences', + }, + ) + ;({ schema } = await schemaRegistry['_getSchema'](registeredSchema.id)) + }) + it('should be able to encode/decode enums schemas', async () => { const obj = { direction: Direction.UP, diff --git a/src/SchemaRegistry.json.spec.ts b/src/SchemaRegistry.json.spec.ts index b2114be..fbade20 100644 --- a/src/SchemaRegistry.json.spec.ts +++ b/src/SchemaRegistry.json.spec.ts @@ -11,7 +11,7 @@ const TestSchemas = { type: SchemaType.JSON, schema: ` { - "$id": "https://sumup.com/schemas/ThirdLevel", + "$id": "https://example.com/schemas/ThirdLevel", "type": "object", "properties": { "id3": { "type": "number" } @@ -24,17 +24,17 @@ const TestSchemas = { type: SchemaType.JSON, schema: ` { - "$id": "https://sumup.com/schemas/SecondLevelA", + "$id": "https://example.com/schemas/SecondLevelA", "type": "object", "properties": { "id2a": { "type": "number" }, - "level2a": { "$ref": "https://sumup.com/schemas/ThirdLevel" } + "level2a": { "$ref": "https://example.com/schemas/ThirdLevel" } } } `, references: [ { - name: 'https://sumup.com/schemas/ThirdLevel', + name: 'https://example.com/schemas/ThirdLevel', subject: 'JSON:ThirdLevel', version: undefined, }, @@ -45,17 +45,17 @@ const TestSchemas = { type: SchemaType.JSON, schema: ` { - "$id": "https://sumup.com/schemas/SecondLevelB", + "$id": "https://example.com/schemas/SecondLevelB", "type": "object", "properties": { "id2b": { "type": "number" }, - "level2b": { "$ref": "https://sumup.com/schemas/ThirdLevel" } + "level2b": { "$ref": "https://example.com/schemas/ThirdLevel" } } } `, references: [ { - name: 'https://sumup.com/schemas/ThirdLevel', + name: 'https://example.com/schemas/ThirdLevel', subject: 'JSON:ThirdLevel', version: undefined, }, @@ -66,23 +66,23 @@ const TestSchemas = { type: SchemaType.JSON, schema: ` { - "$id": "https://sumup.com/schemas/FirstLevel", + "$id": "https://example.com/schemas/FirstLevel", "type": "object", "properties": { "id1": { "type": "number" }, - "level1a": { "$ref": "https://sumup.com/schemas/SecondLevelA" }, - "level1b": { "$ref": "https://sumup.com/schemas/SecondLevelB" } + "level1a": { "$ref": "https://example.com/schemas/SecondLevelA" }, + "level1b": { "$ref": "https://example.com/schemas/SecondLevelB" } } } `, references: [ { - name: 'https://sumup.com/schemas/SecondLevelA', + name: 'https://example.com/schemas/SecondLevelA', subject: 'JSON:SecondLevelA', version: undefined, }, { - name: 'https://sumup.com/schemas/SecondLevelB', + name: 'https://example.com/schemas/SecondLevelB', subject: 'JSON:SecondLevelB', version: undefined, }, @@ -314,16 +314,16 @@ describe('SchemaRegistry', () => { describe('when document example', () => { it('should encode/decode', async () => { const schemaA = { - $id: 'https://sumup.com/schemas/A', + $id: 'https://example.com/schemas/A', type: 'object', properties: { id: { type: 'number' }, - b: { $ref: 'https://sumup.com/schemas/B' }, + b: { $ref: 'https://example.com/schemas/B' }, }, } const schemaB = { - $id: 'https://sumup.com/schemas/B', + $id: 'https://example.com/schemas/B', type: 'object', properties: { id: { type: 'number' }, @@ -344,7 +344,7 @@ describe('SchemaRegistry', () => { schema: JSON.stringify(schemaA), references: [ { - name: 'https://sumup.com/schemas/B', + name: 'https://example.com/schemas/B', subject: 'JSON:B', version, }, diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index 984208d..d665def 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -24,7 +24,8 @@ import { SchemaResponse, ProtocolOptions, SchemaHelper, - ReferenceType, + SchemaReference, + LegacyOptions, } from './@types' import { helperTypeFromSchemaType, @@ -114,7 +115,7 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(confluentSchema.type) - const options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) + const options = await this.updateOptionsWithSchemaReferences(confluentSchema, this.options) const schemaInstance = schemaFromConfluentSchema(confluentSchema, options) helper.validate(schemaInstance) let isFirstTimeRegistration = false @@ -165,39 +166,50 @@ export default class SchemaRegistry { } private async updateOptionsWithSchemaReferences( - options: SchemaRegistryAPIClientOptions | undefined, schema: ConfluentSchema, + options: SchemaRegistryAPIClientOptions | undefined, ) { const helper = helperTypeFromSchemaType(schema.type) - const referredSchemas = await this.getReferredSchemas(schema, helper) - return helper.updateOptionsFromSchemaReferences(options as ProtocolOptions, referredSchemas) + const referencedSchemas = await this.getreferencedSchemas(schema, helper) + + const protocolOptions = this.asProtocolOptions(options) + return helper.updateOptionsFromSchemaReferences(referencedSchemas, protocolOptions) + } + + private asProtocolOptions(options?: SchemaRegistryAPIClientOptions): ProtocolOptions | undefined { + if (!(options as LegacyOptions)?.forSchemaOptions) { + return options as ProtocolOptions | undefined + } + return { + [SchemaType.AVRO]: (options as LegacyOptions)?.forSchemaOptions, + } } - private async getReferredSchemas( + private async getreferencedSchemas( schema: ConfluentSchema, helper: SchemaHelper, ): Promise { const referencesSet = new Set() - return this.getReferredSchemasRecursive(schema, helper, referencesSet) + return this.getreferencedSchemasRecursive(schema, helper, referencesSet) } - private async getReferredSchemasRecursive( + private async getreferencedSchemasRecursive( schema: ConfluentSchema, helper: SchemaHelper, referencesSet: Set, ): Promise { const references = schema.references || [] - let referredSchemas: ConfluentSchema[] = [] + let referencedSchemas: ConfluentSchema[] = [] for (const reference of references) { - const schemas = await this.getReferredSchemasFromReference(reference, helper, referencesSet) - referredSchemas = referredSchemas.concat(schemas) + const schemas = await this.getreferencedSchemasFromReference(reference, helper, referencesSet) + referencedSchemas = referencedSchemas.concat(schemas) } - return referredSchemas + return referencedSchemas } - async getReferredSchemasFromReference( - reference: ReferenceType, + async getreferencedSchemasFromReference( + reference: SchemaReference, helper: SchemaHelper, referencesSet: Set, ): Promise { @@ -214,10 +226,14 @@ export default class SchemaRegistry { const foundSchema = versionResponse.data() as SchemaResponse const schema = helper.toConfluentSchema(foundSchema) - const referredSchemas = await this.getReferredSchemasRecursive(schema, helper, referencesSet) - - referredSchemas.push(schema) - return referredSchemas + const referencedSchemas = await this.getreferencedSchemasRecursive( + schema, + helper, + referencesSet, + ) + + referencedSchemas.push(schema) + return referencedSchemas } private async _getSchema( @@ -237,7 +253,7 @@ export default class SchemaRegistry { const helper = helperTypeFromSchemaType(schemaType) const confluentSchema = helper.toConfluentSchema(foundSchema) - const options = await this.updateOptionsWithSchemaReferences(this.options, confluentSchema) + const options = await this.updateOptionsWithSchemaReferences(confluentSchema, this.options) const schemaInstance = schemaFromConfluentSchema(confluentSchema, options) return this.cache.setSchema(registryId, schemaType, schemaInstance) } From f3c3bdf5f5f4b61478873e32f472d240ee192b92 Mon Sep 17 00:00:00 2001 From: bifrost Date: Mon, 3 Oct 2022 14:30:29 +0200 Subject: [PATCH 22/25] review comments --- src/AvroHelper.ts | 5 ++--- src/JsonHelper.ts | 5 ++--- src/ProtoHelper.ts | 8 +++++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/AvroHelper.ts b/src/AvroHelper.ts index d7e63f8..9f0001b 100644 --- a/src/AvroHelper.ts +++ b/src/AvroHelper.ts @@ -84,9 +84,8 @@ export default class AvroHelper implements SchemaHelper { updateOptionsFromSchemaReferences( referencedSchemas: AvroConfluentSchema[], - options?: ProtocolOptions, + options: ProtocolOptions = {}, ): ProtocolOptions { - const opts = options ?? {} - return { ...opts, [SchemaType.AVRO]: { ...opts[SchemaType.AVRO], referencedSchemas } } + return { ...options, [SchemaType.AVRO]: { ...options[SchemaType.AVRO], referencedSchemas } } } } diff --git a/src/JsonHelper.ts b/src/JsonHelper.ts index 726c89b..06b926b 100644 --- a/src/JsonHelper.ts +++ b/src/JsonHelper.ts @@ -28,9 +28,8 @@ export default class JsonHelper implements SchemaHelper { updateOptionsFromSchemaReferences( referencedSchemas: JsonConfluentSchema[], - options?: ProtocolOptions, + options: ProtocolOptions = {}, ): ProtocolOptions { - const opts = options ?? {} - return { ...opts, [SchemaType.JSON]: { ...opts[SchemaType.JSON], referencedSchemas } } + return { ...options, [SchemaType.JSON]: { ...options[SchemaType.JSON], referencedSchemas } } } } diff --git a/src/ProtoHelper.ts b/src/ProtoHelper.ts index 9ffe8a6..ba46aff 100644 --- a/src/ProtoHelper.ts +++ b/src/ProtoHelper.ts @@ -28,9 +28,11 @@ export default class ProtoHelper implements SchemaHelper { updateOptionsFromSchemaReferences( referencedSchemas: ProtoConfluentSchema[], - options?: ProtocolOptions, + options: ProtocolOptions = {}, ): ProtocolOptions { - const opts = options ?? {} - return { ...opts, [SchemaType.PROTOBUF]: { ...opts[SchemaType.PROTOBUF], referencedSchemas } } + return { + ...options, + [SchemaType.PROTOBUF]: { ...options[SchemaType.PROTOBUF], referencedSchemas }, + } } } From d94bcc40d8b8e88245cad8d3d7e50a65150e7d66 Mon Sep 17 00:00:00 2001 From: bifrost Date: Mon, 3 Oct 2022 14:39:20 +0200 Subject: [PATCH 23/25] review comments --- src/SchemaRegistry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index d665def..f3b4be5 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -167,7 +167,7 @@ export default class SchemaRegistry { private async updateOptionsWithSchemaReferences( schema: ConfluentSchema, - options: SchemaRegistryAPIClientOptions | undefined, + options?: SchemaRegistryAPIClientOptions, ) { const helper = helperTypeFromSchemaType(schema.type) const referencedSchemas = await this.getreferencedSchemas(schema, helper) From d0c9ab9f3d29e74ca5a0f45feeb0dac4fd944fbf Mon Sep 17 00:00:00 2001 From: bifrost Date: Tue, 4 Oct 2022 08:27:20 +0200 Subject: [PATCH 24/25] minor-refactoring --- src/SchemaRegistry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SchemaRegistry.ts b/src/SchemaRegistry.ts index f3b4be5..cb8ce32 100644 --- a/src/SchemaRegistry.ts +++ b/src/SchemaRegistry.ts @@ -246,7 +246,7 @@ export default class SchemaRegistry { } const response = await this.getSchemaOriginRequest(registryId) - const foundSchema = response.data() as SchemaResponse + const foundSchema: SchemaResponse = response.data() const schemaType = schemaTypeFromString(foundSchema.schemaType) From e193664a0e8da29b0583d862c918e64834a9989b Mon Sep 17 00:00:00 2001 From: bifrost Date: Tue, 4 Oct 2022 11:29:22 +0200 Subject: [PATCH 25/25] extend-ajvInstance --- src/@types.ts | 2 ++ src/JsonSchema.ts | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/@types.ts b/src/@types.ts index e4a51af..ad683a0 100644 --- a/src/@types.ts +++ b/src/@types.ts @@ -21,8 +21,10 @@ export interface SchemaHelper { export type AvroOptions = Partial & { referencedSchemas?: AvroConfluentSchema[] } + export type JsonOptions = ConstructorParameters[0] & { ajvInstance?: { + addSchema: Ajv['addSchema'] compile: (schema: any) => ValidateFunction } referencedSchemas?: JsonConfluentSchema[] diff --git a/src/JsonSchema.ts b/src/JsonSchema.ts index e103396..7b0d7cd 100644 --- a/src/JsonSchema.ts +++ b/src/JsonSchema.ts @@ -33,7 +33,6 @@ export default class JsonSchema implements Schema { if (referencedSchemas) { referencedSchemas.forEach(rawSchema => { const $schema = JSON.parse(rawSchema.schema) - // @ts-ignore ajv.addSchema($schema, $schema['$id']) }) }