diff --git a/package-lock.json b/package-lock.json index 02f6b7f..e474504 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,22 +1,25 @@ { "name": "@dylibso/xtp-bindgen", - "version": "1.0.0-rc.6", + "version": "1.0.0-rc.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dylibso/xtp-bindgen", - "version": "1.0.0-rc.6", + "version": "1.0.0-rc.11", "license": "BSD-3-Clause", "devDependencies": { "@extism/js-pdk": "^1.0.1", "@types/jest": "^29.5.12", + "@types/js-yaml": "^4.0.9", + "@types/node": "^22.8.1", "const": "^1.0.0", "esbuild": "^0.17.0", "esbuild-plugin-d.ts": "^1.2.3", "jest": "^29.0.0", + "js-yaml": "^4.1.0", "ts-jest": "^29.0.0", - "typescript": "^5.0.0" + "typescript": "^5.6.3" } }, "node_modules/@ampproject/remapping": { @@ -962,6 +965,28 @@ "node": ">=8" } }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", @@ -1405,13 +1430,19 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true + }, "node_modules/@types/node": { - "version": "20.14.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.9.tgz", - "integrity": "sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg==", + "version": "22.8.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.8.1.tgz", + "integrity": "sha512-k6Gi8Yyo8EtrNtkHXutUu2corfDf9su95VYVP10aGYMMROM6SAItZi0w1XszA6RtWTHSVp5OeFof37w0IEqCQg==", "dev": true, "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.19.8" } }, "node_modules/@types/stack-utils": { @@ -1488,13 +1519,10 @@ } }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "node_modules/babel-jest": { "version": "29.7.0", @@ -3041,13 +3069,12 @@ "dev": true }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -3852,9 +3879,9 @@ } }, "node_modules/typescript": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.2.tgz", - "integrity": "sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==", + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -3865,9 +3892,9 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", "dev": true }, "node_modules/update-browserslist-db": { diff --git a/package.json b/package.json index 8c21e9c..2a1ed9f 100644 --- a/package.json +++ b/package.json @@ -15,12 +15,15 @@ "devDependencies": { "@extism/js-pdk": "^1.0.1", "@types/jest": "^29.5.12", + "@types/js-yaml": "^4.0.9", + "@types/node": "^22.8.1", "const": "^1.0.0", "esbuild": "^0.17.0", "esbuild-plugin-d.ts": "^1.2.3", "jest": "^29.0.0", + "js-yaml": "^4.1.0", "ts-jest": "^29.0.0", - "typescript": "^5.0.0" + "typescript": "^5.6.3" }, "files": [ "dist" diff --git a/src/common.ts b/src/common.ts index 5488b7a..6558623 100644 --- a/src/common.ts +++ b/src/common.ts @@ -1,6 +1,6 @@ -export class ValidationError extends Error { - constructor(public message: string, public path: string) { - super(message); - Object.setPrototypeOf(this, ValidationError.prototype); - } +export class ValidationError { + constructor(public message: string, public path: string) { + this.message = message + this.path = path + } } diff --git a/src/index.ts b/src/index.ts index 3e5212e..7920424 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,9 @@ import { XtpSchema, } from "./normalizer"; import { CodeSample } from "./parser"; +import { XtpNormalizedType } from "./types"; export * from "./normalizer"; +export * from "./types"; export { ValidationError } from "./common"; export function parse(schema: string) { @@ -104,9 +106,25 @@ function isPrimitive(p: Property | Parameter): boolean { return !!p.$ref.enum || !p.$ref.properties; } -function isDateTime(p: Property | Parameter | null): boolean { - if (!p) return false; - return p.type === "string" && p.format === "date-time"; +type XtpTyped = { xtpType: XtpNormalizedType } | null; + +function isDateTime(p: XtpTyped): boolean { + return p?.xtpType?.kind === 'date-time' +} +function isBuffer(p: XtpTyped): boolean { + return p?.xtpType?.kind === "buffer" +} +function isObject(p: XtpTyped): boolean { + return p?.xtpType?.kind === "object" +} +function isArray(p: XtpTyped): boolean { + return p?.xtpType?.kind === "array" +} +function isEnum(p: XtpTyped): boolean { + return p?.xtpType?.kind === "enum" +} +function isString(p: XtpTyped): boolean { + return p?.xtpType?.kind === "string" } function capitalize(s: string) { @@ -135,6 +153,11 @@ export const helpers = { codeSamples, isDateTime, isPrimitive, + isBuffer, + isObject, + isEnum, + isArray, + isString, isJsonEncoded, isUtf8Encoded, capitalize, diff --git a/src/normalizer.ts b/src/normalizer.ts index a12fbcb..1d81604 100644 --- a/src/normalizer.ts +++ b/src/normalizer.ts @@ -1,26 +1,56 @@ +/** + * Normalize reduces the XTP schema down into a simpler, IR-like form. + * + * The primary purpose is to iterate through the raw parsed document + * and normalize away some of the hairy XTP details into our own narrow format. + */ import { ValidationError } from "./common"; import * as parser from "./parser" +import { + XtpNormalizedType, + StringType, ObjectType, EnumType, ArrayType, MapType, + DateTimeType, + Int32Type, + Int64Type, + FloatType, + DoubleType, + BooleanType, + BufferType, +} from "./types" + +export interface XtpTyped extends parser.XtpTyped { + description?: string; + // we are deriving these values + xtpType?: XtpNormalizedType; +} -export interface XtpItemType extends Omit { - '$ref': Schema | null; +export interface Parameter extends parser.Parameter { + // we are deriving these values + xtpType?: XtpNormalizedType; } export interface Property extends Omit { + // we're gonna change this from a string to a Schema object '$ref': Schema | null; - nullable: boolean; + + // we are deriving these values required: boolean; - items?: XtpItemType; name: string; + xtpType?: XtpNormalizedType; } +// TODO fix this? export function isProperty(p: any): p is Property { return !!p.type } export interface Schema extends Omit { properties: Property[]; - additionalProperties?: Property; + additionalProperties?: XtpTyped; name: string; + + // we are deriving these values + xtpType?: XtpNormalizedType; } export type SchemaMap = { @@ -39,7 +69,6 @@ export type Version = 'v0' | 'v1'; export type XtpType = parser.XtpType export type XtpFormat = parser.XtpFormat export type MimeType = parser.MimeType -export type Parameter = parser.Parameter export interface Export { name: string; @@ -49,6 +78,7 @@ export interface Export { output?: Parameter; } +// TODO fix export function isExport(e: any): e is Export { return !!e.name } @@ -76,325 +106,190 @@ function normalizeV0Schema(parsed: parser.V0Schema): XtpSchema { } } -function querySchemaRef(schemas: { [key: string]: Schema }, ref: string, location: string): Schema { - const parts = ref.split('/') - if (parts[0] !== '#') throw new ValidationError("Not a valid ref " + ref, location); - if (parts[1] !== 'components') throw new ValidationError("Not a valid ref " + ref, location); - if (parts[2] !== 'schemas') throw new ValidationError("Not a valid ref " + ref, location); - const name = parts[3]; - - const s = schemas[name] - if (!s) { - const availableSchemas = Object.keys(schemas).join(', ') - throw new ValidationError(`invalid reference ${ref}. Cannot find schema ${name}. Options are: ${availableSchemas}`, location); - } - return s -} +class V1SchemaNormalizer { + version = 'v1' + exports: Export[] = [] + imports: Import[] = [] + schemas: SchemaMap = {} + parsed: parser.V1Schema -function normalizeProp(p: Parameter | Property | XtpItemType | parser.XtpItemType, s: Schema, location: string) { - p.$ref = s - p.description = p.description || s.description - // double ensure that content types are lowercase - if ('contentType' in p) { - p.contentType = p.contentType.toLowerCase() as MimeType - } - if (!p.type) { - p.type = 'string' - } - if (s.type) { - // if it's not an object assume it's a string - if (s.type === 'object') { - p.type = 'object' - } + constructor(parsed: parser.V1Schema) { + this.parsed = parsed } -} -function validateArrayItems(arrayItem: XtpItemType | parser.XtpItemType | undefined, location: string): void { - if (!arrayItem || !arrayItem.type) { - return; - } + normalize(): XtpSchema { + // First let's create all our normalized schemas + // we need these first so we can point $refs to them + for (const name in this.parsed.components?.schemas) { + const pSchema = this.parsed.components.schemas[name] + + // turn any parser.Property map we have into Property[] + const properties = [] + if (pSchema.properties) { + for (const name in pSchema.properties) { + properties.push({ ...pSchema.properties[name], name } as Property) + } + } - validateTypeAndFormat(arrayItem.type, arrayItem.format, location); -} + // we hard cast instead of copy we we can mutate the $refs later + // TODO find a way around this + const schema = (pSchema as unknown) as Schema + schema.name = name + schema.properties = properties + this.schemas[name] = schema + } -function validateTypeAndFormat(type: XtpType, format: XtpFormat | undefined, location: string): void { - const validTypes = ['string', 'number', 'integer', 'boolean', 'object', 'array', 'buffer']; - if (!validTypes.includes(type)) { - throw new ValidationError(`Invalid type '${type}'. Options are: ${validTypes.map(t => `'${t}'`).join(', ')}`, location); - } + // recursively annotate all typed interfaces in the document + this.annotateType(this.parsed as any) - if (!format) { - return; - } + // normalize exports + for (const name in this.parsed.exports) { + const ex = this.parsed.exports[name] as Export + ex.name = name + this.exports.push(ex) + } - let validFormats: XtpFormat[] = []; - if (type === 'string') { - validFormats = ['date-time', 'byte']; - } else if (type === 'number') { - validFormats = ['float', 'double']; - } else if (type === 'integer') { - validFormats = ['int32', 'int64']; - } + // normalize imports + for (const name in this.parsed.imports) { + const im = this.parsed.imports[name] as Import + im.name = name + this.imports.push(im) + } - if (!validFormats.includes(format)) { - throw new ValidationError(`Invalid format ${format} for type ${type}. Valid formats are: ${validFormats.join(', ')}`, location); + return { + version: 'v1', + exports: this.exports, + imports: this.imports, + schemas: this.schemas, + } } -} - -function normalizeV1Schema(parsed: parser.V1Schema): XtpSchema { - const version = 'v1' - const exports: Export[] = [] - const imports: Import[] = [] - const schemas: SchemaMap = {} - - function normalizeMapSchema(s: parser.Schema, schemaName: string): Schema { - const normalizedSchema: Schema = { - ...s, - name: schemaName, - properties: [], - additionalProperties: undefined, - type: 'map', - }; - if (s.additionalProperties) { - if (s.additionalProperties.$ref) { - const refSchema = querySchemaRef(schemas, s.additionalProperties.$ref, `#/components/schemas/${schemaName}/additionalProperties`); - normalizedSchema.additionalProperties = (s as unknown) as Property; - normalizeProp(normalizedSchema.additionalProperties, refSchema, `#/components/schemas/${schemaName}/additionalProperties`); - } else { - normalizedSchema.additionalProperties = s.additionalProperties as Property; - } + querySchemaRef(ref: string, location: string): Schema { + const parts = ref.split('/') + if (parts[0] !== '#') throw new Error("Not a valid ref " + ref); + if (parts[1] !== 'components') throw new Error("Not a valid ref " + ref); + if (parts[2] !== 'schemas') throw new Error("Not a valid ref " + ref); + const name = parts[3]; + + const s = this.schemas[name] + if (!s) { + const availableSchemas = Object.keys(this.schemas).join(', ') + throw new Error(`invalid reference ${ref}. Cannot find schema ${name}. Options are: ${availableSchemas}`); } - return normalizedSchema; + return s } - // need to index all the schemas first - for (const name in parsed.components?.schemas) { - const s = parsed.components.schemas[name] - if (s.additionalProperties) { - schemas[name] = normalizeMapSchema(s, name); - } else if (s.enum) { - schemas[name] = { - ...s, - name, - properties: [], - additionalProperties: undefined, - type: 'enum', - } - } else { - const properties: Property[] = [] - for (const pName in s.properties) { - const p = s.properties[pName] as Property - p.name = pName - properties.push(p) - - if (p.items?.$ref) { - validateArrayItems(p.items, `#/components/schemas/${name}/properties/${pName}/items`); - } - } - - // overwrite the name - // overwrite new properties shape - schemas[name] = { - ...s, - name, - properties, - additionalProperties: undefined, - type: 'object', + // Recursively derive and annotate types + annotateType(s: any): XtpNormalizedType | undefined { + if (!s || typeof s !== 'object' || Array.isArray(s)) return undefined + if (s.xtpType) return s.xtpType // no need to recalculate + + // we can assume this is an object type + if (s.properties && s.properties.length > 0) { + const properties: XtpNormalizedType[] = [] + for (const pname in s.properties!) { + const p = s.properties[pname] + const t = this.annotateType(p)! + p.xtpType = t + properties.push(t) } - } - } - // denormalize all the properties in a second loop - for (const name in schemas) { - const s = schemas[name] - - s.properties?.forEach((p, idx) => { - // link the property with a reference to the schema if it has a ref - // need to get the ref from the parsed (raw) property - const rawProp = parsed.components!.schemas![name].properties![p.name] - const propPath = `#/components/schemas/${name}/properties/${p.name}`; - - if (rawProp.$ref) { - normalizeProp( - schemas[name].properties[idx], - querySchemaRef(schemas, rawProp.$ref, propPath), - propPath - ) - } + // TODO remove this legacy code + // we need to derive old type here + s.type = 'object' - if (rawProp.items?.$ref) { - const path = `${propPath}/items` + return new ObjectType(s.name!, properties, s) + } - normalizeProp( - p.items!, - querySchemaRef(schemas, rawProp.items!.$ref, path), - path - ) + if (s.$ref) { + let ref = s.$ref + // this conditional takes the place of all the legacy code to replace + // the sring with the ref + if (typeof s.$ref === 'string') { + ref = this.querySchemaRef(s.$ref, '') + s.$ref = ref } - validateTypeAndFormat(p.type, p.format, propPath); - validateArrayItems(p.items, `${propPath}/items`); + // TODO remove this legacy code + // we need to derive old type here + s.type = 'object' - // coerce to false by default for nullable and required - p.nullable = p.nullable || false - p.required = !!s.required?.includes(p.name) - }) - } - - // denormalize all the exports - for (const name in parsed.exports) { - let ex = parsed.exports[name] - - const normEx = ex as Export - normEx.name = name - - if (ex.input?.$ref) { - const path = `#/exports/${name}/input` - - normalizeProp( - normEx.input!, - querySchemaRef(schemas, ex.input.$ref, path), - path - ) - } - if (ex.input?.items?.$ref) { - const path = `#/exports/${name}/input/items` - - normalizeProp( - normEx.input!.items!, - querySchemaRef(schemas, ex.input.items.$ref, path), - path - ) + return this.annotateType(ref)! } - if (ex.output?.$ref) { - const path = `#/exports/${name}/output` + // enums can only be string enums right now + if (s.enum) { + // TODO remove this legacy code + // we need to derive old type here + s.type = 'enum' - normalizeProp( - normEx.output!, - querySchemaRef(schemas, ex.output.$ref, path), - path - ) - } - if (ex.output?.items?.$ref) { - const path = `#/exports/${name}/output/items` - - normalizeProp( - normEx.output!.items!, - querySchemaRef(schemas, ex.output.items.$ref, path), - path - ) + return new EnumType(s.name!, new StringType(), s.enum, s) } - validateArrayItems(normEx.input?.items, `#/exports/${name}/input/items`); - validateArrayItems(normEx.output?.items, `#/exports/${name}/output/items`); - - exports.push(normEx) - } - - // denormalize all the imports - for (const name in parsed.imports) { - const im = parsed.imports![name] - - // they have the same type - const normIm = im as Import - normIm.name = name - - // deref input and output - if (im.input?.$ref) { - const path = `#/imports/${name}/input` - - normalizeProp( - normIm.input!, - querySchemaRef(schemas, im.input.$ref, path), - path - ) - } - if (im.input?.items?.$ref) { - const path = `#/imports/${name}/input/items` - - normalizeProp( - normIm.input!.items!, - querySchemaRef(schemas, im.input.items.$ref, path), - path - ) + // if items is present it's an array + if (s.items) { + return new ArrayType(this.annotateType(s.items)!, s) } - if (im.output?.$ref) { - const path = `#/imports/${name}/output` + // if additionalProperties is present it's a map + if (s.additionalProperties) { + // TODO remove this legacy code + // we need to derive old type here + s.type = 'map' - normalizeProp( - normIm.output!, - querySchemaRef(schemas, im.output.$ref, path), - path - ) + return new MapType(this.annotateType(s.additionalProperties)!, s) } - if (im.output?.items?.$ref) { - const path = `#/imports/${name}/output/items` - - normalizeProp( - normIm.output!.items!, - querySchemaRef(schemas, im.output.items.$ref, path), - path - ) - } - - validateArrayItems(normIm.input?.items, `#/imports/${name}/input/items`); - validateArrayItems(normIm.output?.items, `#/imports/${name}/output/items`); - imports.push(normIm) - } + switch (s.type) { + case 'string': + if (s.format === 'date-time') return new DateTimeType(s) + return new StringType(s) + case 'integer': + return new Int32Type(s) + case 'boolean': + return new BooleanType(s) + case 'buffer': + return new BufferType(s) + case 'number': + if (s.format === 'int32') return new Int32Type(s) + if (s.format === 'int64') return new Int64Type(s) + if (s.format === 'float') return new FloatType(s) + if (s.format === 'double') return new DoubleType(s) + throw new Error(`IDK how to parse this number ${JSON.stringify(s)}`) + } - for (const name in schemas) { - const schema = schemas[name] - const error = detectCircularReference(schema); - if (error) { - throw error; + // if we get this far, we don't know what + // this node is let's just keep drilling down + for (const key in s) { + if (Object.prototype.hasOwnProperty.call(s, key)) { + const child = s[key] + if (child && typeof child === 'object' && !Array.isArray(child)) { + this.annotateType(child); + } + } } } +} - return { - version, - exports, - imports, - schemas, - } +function normalizeV1Schema(parsed: parser.V1Schema): XtpSchema { + const normalizer = new V1SchemaNormalizer(parsed) + return normalizer.normalize() } export function parseAndNormalizeJson(encoded: string): XtpSchema { - const parsed = parser.parseJson(encoded) + const { doc, errors } = parser.parseAny(JSON.parse(encoded)) - if (parser.isV0Schema(parsed)) { - return normalizeV0Schema(parsed) - } else if (parser.isV1Schema(parsed)) { - return normalizeV1Schema(parsed) - } else { - throw new ValidationError("Could not normalize unknown version of schema", "#"); - } -} - -function detectCircularReference(schema: Schema, visited: Set = new Set()): ValidationError | null { - if (visited.has(schema.name)) { - return new ValidationError("Circular reference detected", `#/components/schemas/${schema.name}`); + if (errors && errors.length > 0) { + console.log(JSON.stringify(errors)) + throw Error(`Invalid document`) } - visited.add(schema.name); - - for (const property of schema.properties) { - if (property.$ref) { - const error = detectCircularReference(property.$ref, new Set(visited)); - if (error) { - return error; - } - } else if (property.items?.$ref) { - const error = detectCircularReference(property.items.$ref, new Set(visited)); - if (error) { - return error; - } - } + if (parser.isV0Schema(doc)) { + return normalizeV0Schema(doc) + } else if (parser.isV1Schema(doc)) { + return normalizeV1Schema(doc) + } else { + throw new Error("Could not normalize unknown version of schema"); } - - return null; } diff --git a/src/parser.ts b/src/parser.ts index c0235bd..9db9b7c 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,4 +1,143 @@ -import { ValidationError } from "./common"; +/** + * The parser is responsible for taking a raw JS object + * of the XTP Schema and parsing and typing it without transforming + * any of the values. It's just mean to give a typed representation + * of the raw format. The one exception is we do ammend the major + * nodes with a `nodeKind` property to make it easier to identify + * what node we're looking at in-situ at runtime. We have 1 type + * per version of the schema. + * + * Call `parseJson(doc: string)` to parse the doc into a version agnostic + * type. Then discriminate with isV0Schema or isV1Schema. + */ +import { ValidationError } from "./common" + +export interface ParseResult { + doc?: VUnknownSchema; + errors?: ValidationError[]; +} + +/** + * Parses and validates an untyped object into a V*Schema + */ +export function parseAny(doc: any): ParseResult { + switch (doc.version) { + case 'v0': + return { doc: doc as V0Schema } + case 'v1-draft': + const v1Doc = doc as V1Schema + setNodeKinds(v1Doc) + const validator = new V1Validator(v1Doc) + const errors = validator.validate() + return { doc: v1Doc, errors } + default: + return { + errors: [ + new ValidationError(`version property not valid: ${doc.version}`, "#/version") + ] + } + } +} + +export function isV0Schema(schema?: VUnknownSchema): schema is V0Schema { + return schema?.version === 'v0'; +} + +export function isV1Schema(schema?: VUnknownSchema): schema is V1Schema { + return schema?.version === 'v1-draft'; +} + +/** + * Validates a V1 document. + */ +class V1Validator { + errors: ValidationError[] + location: string[] + doc: any + + constructor(doc: V1Schema) { + this.doc = doc as any + this.errors = [] + this.location = ['#'] + } + + /** + * Validate the document and return any errors + */ + validate(): ValidationError[] { + this.errors = [] + this.validateNode(this.doc) + return this.errors + } + + /** + * Recursively walk through the untyped document and validate each node. + * This saves us a lot of code but might be a little bit slower. + */ + validateNode(node: any) { + this.validateTypedInterface(node) + if (node && typeof node === 'object') { + // i don't think we need to validate array children + if (Array.isArray(node)) return + + for (const key in node) { + if (Object.prototype.hasOwnProperty.call(node, key)) { + const child = node[key] + if (typeof child === 'object') { + this.location.push(key) + this.validateNode(child); + this.location.pop() + } + } + } + } + } + + recordError(msg: string) { + this.errors.push( + new ValidationError(msg, this.getLocation()) + ) + } + + /** + * Validates that a node conforms to the rules of + * the XtpTyped interface. Validates what we can't + * catch in JSON Schema validation. + */ + validateTypedInterface(prop?: XtpTyped): void { + if (!prop || !prop.type) return + + const validTypes = ['string', 'number', 'integer', 'boolean', 'object', 'array', 'buffer']; + if (!validTypes.includes(prop.type)) { + this.recordError(`Invalid type '${prop.type}'. Options are: ${validTypes.map(t => `'${t}'`).join(', ')}`) + } + + if (!prop.format) { + return; + } + + let validFormats: XtpFormat[] = []; + if (prop.type === 'string') { + validFormats = ['date-time', 'byte']; + } else if (prop.type === 'number') { + validFormats = ['float', 'double']; + } else if (prop.type === 'integer') { + validFormats = ['int32', 'int64']; + } + + if (!validFormats.includes(prop.format)) { + this.recordError(`Invalid format ${prop.format} for type ${prop.type}. Valid formats are: ${validFormats.join(', ')}`) + } + + if (prop.items) this.validateTypedInterface(prop.items) + if (prop.additionalProperties) this.validateTypedInterface(prop.additionalProperties) + } + + getLocation(): string { + return this.location.join('/') + } +} + // Main Schema export interface export interface V0Schema { @@ -15,6 +154,9 @@ export interface V1Schema { } } +// These are the only types we're interested in discriminating +export type NodeKind = 'schema' | 'property' | 'parameter' | 'import' | 'export' + type VUnknownSchema = V0Schema | V1Schema export type Version = 'v0' | 'v1-draft'; @@ -25,6 +167,8 @@ export type Import = Export export type SimpleExport = string; export interface Export { + nodeKind: NodeKind; + name: string; description?: string; codeSamples?: CodeSample[]; @@ -40,71 +184,70 @@ export interface CodeSample { export type MimeType = 'application/json' | 'text/plain; charset=utf-8' | 'application/x-binary' -export interface Schema { - description: string; - type?: XtpSchemaType; - enum?: string[]; - properties?: { [name: string]: Property }; - additionalProperties?: Property; +export interface Schema extends XtpTyped { + description?: string; + nodeKind: NodeKind; required?: string[]; + properties?: { [name: string]: Property }; } -export type XtpSchemaType = 'object' | 'enum' | 'map' +// TODO this figure out how to split up type again? +//export type XtpSchemaType = 'object' | 'enum' | 'map' + export type XtpType = - 'integer' | 'string' | 'number' | 'boolean' | 'object' | 'array' | 'buffer'; + 'integer' | 'string' | 'number' | 'boolean' | 'object' | + 'array' | 'buffer' | 'object' | 'enum' | 'map'; export type XtpFormat = 'int32' | 'int64' | 'float' | 'double' | 'date-time' | 'byte'; -export interface XtpItemType { - type: XtpType; + +// Shared interface for any place you can +// define some types inline. Ex: Schema, Property, Parameter +export interface XtpTyped { + type?: XtpType; format?: XtpFormat; + items?: XtpTyped; + additionalProperties?: Property; + nullable?: boolean; + enum?: string[]; + name?: string; + //properties?: { [name: string]: Property }; + // NOTE: needs to be any to satisfy type satisfy - // type system in normalizer + // type system in normalizer, but is in fact a string at this layer "$ref"?: any; - description?: string; +} - // we only support one nested item type for now - // type: XtpType | XtpItemType; +// A property is a named sub-property of an object +// It's a mostly type info +export interface Property extends XtpTyped { + nodeKind: NodeKind; + description?: string; } +// The input and output of boundary functions are Parameters. +// It's a mostly a Property, but also has a required encoding. export interface Parameter extends Property { + nodeKind: NodeKind; contentType: MimeType; } -export interface Property { - type: XtpType; - items?: XtpItemType; - format?: XtpFormat; - description?: string; - nullable?: boolean; - - // NOTE: needs to be any to satisfy type safity in normalizer - "$ref"?: any; -} - -export function parseJson(encoded: string): VUnknownSchema { - let parsed: any; - try { - parsed = JSON.parse(encoded); - } catch (e) { - throw new ValidationError("Invalid JSON", "#"); +// internal helper to set the node nodeKind names in the appropriate places +function setNodeKinds(doc: V1Schema) { + const setFuncNodes = (f: Import | Export, k: NodeKind) => { + f.nodeKind = k + if (f.input) f.input.nodeKind = 'parameter' + if (f.output) f.output.nodeKind = 'parameter' } - - if (!parsed.version) throw new ValidationError("version property missing", "#"); - switch (parsed.version) { - case 'v0': - return parsed as V0Schema; - case 'v1-draft': - return parsed as V1Schema; - default: - throw new ValidationError(`version property not valid: ${parsed.version}`, "#/version"); + for (const eName in doc.exports) setFuncNodes(doc.exports[eName], 'export') + for (const iName in doc.imports) setFuncNodes(doc.imports[iName], 'import') + + for (const sName in doc.components?.schemas) { + const s = doc.components.schemas[sName] + s.nodeKind = 'schema' + for (const pName in s.properties || {}) { + s.properties![pName].nodeKind = 'property' + } } } -export function isV0Schema(schema: VUnknownSchema): schema is V0Schema { - return schema.version === 'v0'; -} - -export function isV1Schema(schema: VUnknownSchema): schema is V1Schema { - return schema.version === 'v1-draft'; -} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..d76f062 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,136 @@ +/** + * These represent the types in our abstract type system. + * We will normalize the raw XTP schema into these recursively defined types. + */ + +export type XtpNormalizedKind = + 'object' | 'enum' | 'map' | 'array' | 'string' | + 'int32' | 'int64' | 'float' | 'double' | + 'boolean' | 'date-time' | 'byte' | 'buffer' + +// applies type opts to a type on construction +function cons(t: XtpNormalizedType, opts?: XtpTypeOpts): XtpNormalizedType { + t.nullable = opts?.nullable + t.required = opts?.required + return t +} + +export interface XtpTypeOpts { + nullable?: boolean; + required?: boolean; +} + +export interface XtpNormalizedType extends XtpTypeOpts { + kind: XtpNormalizedKind; +} + +export class StringType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'string'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class Int32Type implements XtpNormalizedType { + kind: XtpNormalizedKind = 'int32'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class Int64Type implements XtpNormalizedType { + kind: XtpNormalizedKind = 'int64'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class FloatType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'float'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class DoubleType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'double'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class BooleanType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'boolean'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class ByteType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'byte'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class BufferType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'buffer'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class DateTimeType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'date-time'; + constructor(opts?: XtpTypeOpts) { + cons(this, opts) + } +} + +export class ObjectType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'object'; + name: string; + properties: Array; + + constructor(name: string, properties: Array, opts?: XtpTypeOpts) { + this.name = name + this.properties = properties + cons(this, opts) + } +} + +export class MapType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'map'; + keyType = new StringType(); // strings only for now + valueType: XtpNormalizedType; + + constructor(valueType: XtpNormalizedType, opts?: XtpTypeOpts) { + this.valueType = valueType + cons(this, opts) + } +} + +export class EnumType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'enum'; + name: string; + elementType: XtpNormalizedType; + values: any[]; + + constructor(name: string, elementType: XtpNormalizedType, values: any[], opts?: XtpTypeOpts) { + this.name = name + this.elementType = elementType + this.values = values + cons(this, opts) + } +} + +export class ArrayType implements XtpNormalizedType { + kind: XtpNormalizedKind = 'array'; + elementType: XtpNormalizedType; + + constructor(elementType: XtpNormalizedType, opts?: XtpTypeOpts) { + this.elementType = elementType + cons(this, opts) + } +} + diff --git a/tests/index.test.ts b/tests/index.test.ts index 21f262a..85feb40 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,116 +1,13 @@ import { parse } from '../src/index'; +import * as yaml from 'js-yaml' +import * as fs from 'fs' -const testSchema = { - "version": "v1-draft", - "exports": { - "voidFunc": { - "description": "This demonstrates how you can create an export with\nno inputs or outputs.\n" - }, - "primitiveTypeFunc": { - "description": "This demonstrates how you can accept or return primtive types.\nThis function takes a utf8 string and returns a json encoded boolean\n", - "input": { - "type": "string", - "description": "A string passed into plugin input", - "contentType": "text/plain; charset=utf-8" - }, - "output": { - "type": "boolean", - "description": "A boolean encoded as json", - "contentType": "application/json" - }, - "codeSamples": [ - { - "lang": "typescript", - "label": "Test if a string has more than one character.\nCode samples show up in documentation and inline in docstrings\n", - "source": "function primitiveTypeFunc(input: string): boolean {\n return input.length > 1\n}\n" - } - ] - }, - "referenceTypeFunc": { - "description": "This demonstrates how you can accept or return references to schema types.\nAnd it shows how you can define an enum to be used as a property or input/output.\n", - "input": { - "contentType": "application/json", - "$ref": "#/components/schemas/Fruit" - }, - "output": { - "contentType": "application/json", - "$ref": "#/components/schemas/ComplexObject" - } - } - }, - "imports": { - "eatAFruit": { - "description": "This is a host function. Right now host functions can only be the type (i64) -> i64.\nWe will support more in the future. Much of the same rules as exports apply.\n", - "input": { - "contentType": "text/plain; charset=utf-8", - "$ref": "#/components/schemas/Fruit" - }, - "output": { - "type": "boolean", - "description": "boolean encoded as json", - "contentType": "application/json" - } - } - }, - "components": { - "schemas": { - "Fruit": { - "description": "A set of available fruits you can consume", - "enum": [ - "apple", - "orange", - "banana", - "strawberry" - ] - }, - "GhostGang": { - "description": "A set of all the enemies of pac-man", - "enum": [ - "blinky", - "pinky", - "inky", - "clyde" - ] - }, - "ComplexObject": { - "description": "A complex json object", - "properties": { - "ghost": { - "$ref": "#/components/schemas/GhostGang", - "description": "I can override the description for the property here" - }, - "aBoolean": { - "type": "boolean", - "description": "A boolean prop" - }, - "aString": { - "type": "string", - "description": "An string prop" - }, - "anInt": { - "type": "integer", - "format": "int32", - "description": "An int prop" - }, - "anOptionalDate": { - "type": "string", - "format": "date-time", - "description": "A datetime object, we will automatically serialize and deserialize\nthis for you.", - "nullable": true - } - } - }, - "MapSchema": { - "additionalProperties": { - "type": "string" - } - }, - } - } -} +const validV1Doc: any = yaml.load(fs.readFileSync('./tests/schemas/v1-valid-doc.yaml', 'utf8')) test('parse-v1-document', () => { - const doc = parse(JSON.stringify(testSchema)) + const doc = parse(JSON.stringify(validV1Doc)) + + //console.log(JSON.stringify(doc, null, 4)) // check top level document is correct expect(doc.version).toBe('v1') @@ -120,18 +17,18 @@ test('parse-v1-document', () => { const enumSchema1 = doc.schemas['Fruit'] expect(enumSchema1.type).toBe('enum') - expect(enumSchema1.enum).toStrictEqual(testSchema.components.schemas['Fruit'].enum) + expect(enumSchema1.enum).toStrictEqual(validV1Doc.components.schemas['Fruit'].enum) const enumSchema2 = doc.schemas['GhostGang'] expect(enumSchema2.type).toBe('enum') - expect(enumSchema2.enum).toStrictEqual(testSchema.components.schemas['GhostGang'].enum) + expect(enumSchema2.enum).toStrictEqual(validV1Doc.components.schemas['GhostGang'].enum) const schema3 = doc.schemas['ComplexObject'] expect(schema3.type).toBe('object') const properties = schema3.properties // proves we derferenced it - expect(properties[0].$ref?.enum).toStrictEqual(testSchema.components.schemas['GhostGang'].enum) + expect(properties[0].$ref?.enum).toStrictEqual(validV1Doc.components.schemas['GhostGang'].enum) expect(properties[0].$ref?.name).toBe('GhostGang') expect(properties[0].name).toBe('ghost') @@ -141,7 +38,7 @@ test('parse-v1-document', () => { const exp = doc.exports[2] // proves we derferenced it - expect(exp.input?.$ref?.enum).toStrictEqual(testSchema.components.schemas['Fruit'].enum) + expect(exp.input?.$ref?.enum).toStrictEqual(validV1Doc.components.schemas['Fruit'].enum) expect(exp.output?.contentType).toBe('application/json') }) -3 \ No newline at end of file + diff --git a/tests/parser.test.ts b/tests/parser.test.ts new file mode 100644 index 0000000..d748977 --- /dev/null +++ b/tests/parser.test.ts @@ -0,0 +1,41 @@ +import { parseAny, V1Schema } from '../src/parser'; +import * as yaml from 'js-yaml' +import * as fs from 'fs' + +const invalidV1Doc: any = yaml.load(fs.readFileSync('./tests/schemas/v1-invalid-doc.yaml', 'utf8')) +const validV1Doc: any = yaml.load(fs.readFileSync('./tests/schemas/v1-valid-doc.yaml', 'utf8')) + +test("parse-empty-v1-document", () => { + const { errors } = parseAny({}) + expect(errors).toBeInstanceOf(Array) + + expect(errors![0].path).toEqual("#/version") +}) + +test("parse-invalid-v1-document", () => { + const { errors } = parseAny(invalidV1Doc) + expect(errors).toBeInstanceOf(Array) + + const paths = errors!.map(e => e.path) + expect(paths).toStrictEqual([ + "#/exports/invalidFunc1/input", + "#/exports/invalidFunc1/output", + "#/components/schemas/ComplexObject/properties/aBoolean", + "#/components/schemas/ComplexObject/properties/aString", + "#/components/schemas/ComplexObject/properties/anInt", + "#/components/schemas/ComplexObject/properties/aNonType", + ]) +}) + +test("parse-valid-v1-document", () => { + const { doc, errors } = parseAny(validV1Doc) + expect(errors).toStrictEqual([]) + + const schema = doc as V1Schema + + expect(schema.version).toEqual('v1-draft') + expect(schema.exports['primitiveTypeFunc']?.input?.nodeKind).toEqual('parameter') + expect(schema.exports['primitiveTypeFunc']?.output?.nodeKind).toEqual('parameter') + expect(schema?.components?.schemas?.ComplexObject?.nodeKind).toEqual('schema') + expect(schema?.components?.schemas?.ComplexObject?.properties?.ghost?.nodeKind).toEqual('property') +}) diff --git a/tests/schemas/v1-invalid-doc.yaml b/tests/schemas/v1-invalid-doc.yaml new file mode 100644 index 0000000..02177f4 --- /dev/null +++ b/tests/schemas/v1-invalid-doc.yaml @@ -0,0 +1,42 @@ +--- +version: v1-draft +exports: + invalidFunc1: + description: Has some invalid parameters + input: + type: buffer + format: date-time + output: + type: string + format: float +components: + schemas: + GhostGang: + description: a set of all the enemies of pac-man + enum: + - blinky + - pinky + - inky + - clyde + ComplexObject: + description: a complex json object + properties: + ghost: + "$ref": "#/components/schemas/GhostGang" + description: i can override the description for the property here + aBoolean: + type: boolean + format: date-time + description: a boolean prop + aString: + type: string + format: int32 + description: an string prop + anInt: + type: integer + format: date-time + description: an int prop + aNonType: + type: non + description: an int prop + diff --git a/tests/schemas/v1-valid-doc.yaml b/tests/schemas/v1-valid-doc.yaml new file mode 100644 index 0000000..792aaee --- /dev/null +++ b/tests/schemas/v1-valid-doc.yaml @@ -0,0 +1,93 @@ +--- +version: v1-draft +exports: + voidFunc: + description: | + This demonstrates how you can create an export with + no inputs or outputs. + primitiveTypeFunc: + description: | + This demonstrates how you can accept or return primtive types. + This function takes a utf8 string and returns a json encoded boolean + input: + type: string + description: A string passed into plugin input + contentType: text/plain; charset=utf-8 + output: + type: boolean + description: A boolean encoded as json + contentType: application/json + codeSamples: + - lang: typescript + label: | + Test if a string has more than one character. + Code samples show up in documentation and inline in docstrings + source: | + function primitiveTypeFunc(input: string): boolean { + return input.length > 1 + } + referenceTypeFunc: + description: | + This demonstrates how you can accept or return references to schema types. + And it shows how you can define an enum to be used as a property or input/output. + input: + contentType: application/json + "$ref": "#/components/schemas/Fruit" + output: + contentType: application/json + "$ref": "#/components/schemas/ComplexObject" +imports: + eatAFruit: + description: | + This is a host function. Right now host functions can only be the type (i64) -> i64. + We will support more in the future. Much of the same rules as exports apply. + input: + contentType: text/plain; charset=utf-8 + "$ref": "#/components/schemas/Fruit" + output: + type: boolean + description: boolean encoded as json + contentType: application/json +components: + schemas: + Fruit: + description: A set of available fruits you can consume + enum: + - apple + - orange + - banana + - strawberry + GhostGang: + description: A set of all the enemies of pac-man + enum: + - blinky + - pinky + - inky + - clyde + ComplexObject: + description: A complex json object + properties: + ghost: + "$ref": "#/components/schemas/GhostGang" + description: I can override the description for the property here + aBoolean: + type: boolean + description: A boolean prop + aString: + type: string + description: An string prop + anInt: + type: integer + format: int32 + description: An int prop + anOptionalDate: + type: string + format: date-time + description: |- + A datetime object, we will automatically serialize and deserialize + this for you. + nullable: true + MapSchema: + additionalProperties: + type: string + diff --git a/tsconfig.json b/tsconfig.json index 8c7b6d4..830ee21 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,7 +15,8 @@ "allowSyntheticDefaultImports": true, "types": [ "@extism/js-pdk", - "jest" + "jest", + "node" ] }, "include": [