From a79b0c9d07fdda8874fb77585c4d6b1ff23ac344 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 8 Jan 2024 16:35:34 +0100 Subject: [PATCH 01/27] start debugging dex example --- src/examples/zkapps/dex/dex.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/examples/zkapps/dex/dex.ts b/src/examples/zkapps/dex/dex.ts index 8bf5a1ea8f..0f81f0a0c3 100644 --- a/src/examples/zkapps/dex/dex.ts +++ b/src/examples/zkapps/dex/dex.ts @@ -441,16 +441,16 @@ class TokenContract extends SmartContract { amount: UInt64 ) { // TODO: THIS IS INSECURE. The proper version has a prover error (compile != prove) that must be fixed - this.approve(zkappUpdate, AccountUpdate.Layout.AnyChildren); + // this.approve(zkappUpdate, AccountUpdate.Layout.AnyChildren); // THIS IS HOW IT SHOULD BE DONE: - // // approve a layout of two grandchildren, both of which can't inherit the token permission - // let { StaticChildren, AnyChildren } = AccountUpdate.Layout; - // this.approve(zkappUpdate, StaticChildren(AnyChildren, AnyChildren)); - // zkappUpdate.body.mayUseToken.parentsOwnToken.assertTrue(); - // let [grandchild1, grandchild2] = zkappUpdate.children.accountUpdates; - // grandchild1.body.mayUseToken.inheritFromParent.assertFalse(); - // grandchild2.body.mayUseToken.inheritFromParent.assertFalse(); + // approve a layout of two grandchildren, both of which can't inherit the token permission + let { StaticChildren, AnyChildren } = AccountUpdate.Layout; + this.approve(zkappUpdate, StaticChildren(AnyChildren, AnyChildren)); + zkappUpdate.body.mayUseToken.parentsOwnToken.assertTrue(); + let [grandchild1, grandchild2] = zkappUpdate.children.accountUpdates; + grandchild1.body.mayUseToken.inheritFromParent.assertFalse(); + grandchild2.body.mayUseToken.inheritFromParent.assertFalse(); // see if balance change cancels the amount sent let balanceChange = Int64.fromObject(zkappUpdate.body.balanceChange); From 01595e119e6695c5e9492d7ddb8a1d8846f77994 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 8 Jan 2024 22:13:32 +0100 Subject: [PATCH 02/27] start adding types for low level snarky intf --- src/snarky.d.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/snarky.d.ts b/src/snarky.d.ts index 2c47b9dc63..40cb0ed7c7 100644 --- a/src/snarky.d.ts +++ b/src/snarky.d.ts @@ -540,8 +540,31 @@ declare const Snarky: { squeeze(sponge: unknown): FieldVar; }; }; + + lowLevel: { + fieldVec(): unknown; + getState(): SnarkyState; + }; }; +type Ref = [0, T]; + +type Vector = unknown; + +type SnarkyState = [ + _: 0, + system: MlOption, + input: Vector, + aux: Vector, + eval_constraints: MlBool, + num_inputs: number, + next_auxiliary: Ref, + has_witness: MlBool, + stack: MlList, + is_running: MlBool, + log_constraint: unknown +]; + type GateType = | 'Zero' | 'Generic' From b26f2b21abe1c4f539b18216f1656c23b18ba843 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 9 Jan 2024 16:16:35 +0100 Subject: [PATCH 03/27] iterate on types --- src/snarky.d.ts | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/snarky.d.ts b/src/snarky.d.ts index 40cb0ed7c7..2ba81fb333 100644 --- a/src/snarky.d.ts +++ b/src/snarky.d.ts @@ -26,6 +26,7 @@ import type { WasmFqSrs, } from './bindings/compiled/node_bindings/plonk_wasm.cjs'; import type { KimchiGateType } from './lib/gates.ts'; +import type { FieldVector } from './bindings/crypto/bindings/vector.ts'; export { ProvablePure, Provable, Ledger, Pickles, Gate, GateType, getWasm }; @@ -542,20 +543,38 @@ declare const Snarky: { }; lowLevel: { - fieldVec(): unknown; - getState(): SnarkyState; + state: Ref; + setState(state: SnarkyState): void; + createState( + numInputs: number, + evalConstraints: MlBool, + withWitness: MlBool, + logConstraint: MlOption< + ( + atLabelBoundary: MlOption, + constraint: MlOption + ) => void + > + ): [ + _: 0, + state: SnarkyState, + input: FieldVector, + aux: FieldVector, + system: ConstraintSystem + ]; }; }; -type Ref = [0, T]; +type Ref = [_: 0, contents: T]; -type Vector = unknown; +type SnarkyVector = [0, [unknown, number, FieldVector]]; +type ConstraintSystem = unknown; type SnarkyState = [ _: 0, - system: MlOption, - input: Vector, - aux: Vector, + system: MlOption, + input: SnarkyVector, + aux: SnarkyVector, eval_constraints: MlBool, num_inputs: number, next_auxiliary: Ref, @@ -565,6 +584,12 @@ type SnarkyState = [ log_constraint: unknown ]; +type SnarkyConstraint = [ + _: 0, + basic: [number, unknown], // actually this is an enum + annotation: MlOption +]; + type GateType = | 'Zero' | 'Generic' From 7444829949989c337a3a15decdb591b830911261 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 10:07:49 +0100 Subject: [PATCH 04/27] iterate on api and fix type error --- src/snarky.d.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/snarky.d.ts b/src/snarky.d.ts index 2ba81fb333..3657f3685a 100644 --- a/src/snarky.d.ts +++ b/src/snarky.d.ts @@ -39,6 +39,8 @@ export { MlPublicKeyVar, FeatureFlags, MlFeatureFlags, + SnarkyState, + SnarkyConstraint, }; /** @@ -543,8 +545,7 @@ declare const Snarky: { }; lowLevel: { - state: Ref; - setState(state: SnarkyState): void; + state: MlRef; createState( numInputs: number, evalConstraints: MlBool, @@ -562,10 +563,13 @@ declare const Snarky: { aux: FieldVector, system: ConstraintSystem ]; + + pushActiveCounter(): MlList; + resetActiveCounter(counters: MlList): void; }; }; -type Ref = [_: 0, contents: T]; +type MlRef = [_: 0, contents: T]; type SnarkyVector = [0, [unknown, number, FieldVector]]; type ConstraintSystem = unknown; @@ -577,10 +581,12 @@ type SnarkyState = [ aux: SnarkyVector, eval_constraints: MlBool, num_inputs: number, - next_auxiliary: Ref, + next_auxiliary: MlRef, has_witness: MlBool, stack: MlList, + handler: unknown, is_running: MlBool, + as_prover: MlRef, log_constraint: unknown ]; From bb54d4f9ac4523a03c18038f449dbab4608aaabe Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 12:45:27 +0100 Subject: [PATCH 05/27] runner that can compare circuit constraints against a previous run --- src/lib/gates.ts | 30 ++++ src/lib/provable-context-debug.ts | 175 ++++++++++++++++++++ src/lib/provable-context-debug.unit-test.ts | 16 ++ src/lib/util/nested.ts | 44 +++++ src/snarky.d.ts | 8 +- 5 files changed, 266 insertions(+), 7 deletions(-) create mode 100644 src/lib/provable-context-debug.ts create mode 100644 src/lib/provable-context-debug.unit-test.ts create mode 100644 src/lib/util/nested.ts diff --git a/src/lib/gates.ts b/src/lib/gates.ts index a8900cfe49..35154986a5 100644 --- a/src/lib/gates.ts +++ b/src/lib/gates.ts @@ -15,6 +15,8 @@ export { foreignFieldAdd, foreignFieldMul, KimchiGateType, + KimchiGateTypeString, + gateTypeToString, }; const Gates = { @@ -265,3 +267,31 @@ enum KimchiGateType { Xor16, Rot64, } + +function gateTypeToString(gate: KimchiGateType) { + return KimchiGateTypeToString[gate]; +} + +type KimchiGateTypeString = + (typeof KimchiGateTypeToString)[keyof typeof KimchiGateTypeToString]; + +const KimchiGateTypeToString = { + [KimchiGateType.Zero]: 'Zero', + [KimchiGateType.Generic]: 'Generic', + [KimchiGateType.Poseidon]: 'Poseidon', + [KimchiGateType.CompleteAdd]: 'CompleteAdd', + [KimchiGateType.VarBaseMul]: 'VarBaseMul', + [KimchiGateType.EndoMul]: 'EndoMul', + [KimchiGateType.EndoMulScalar]: 'EndoMulScalar', + [KimchiGateType.Lookup]: 'Lookup', + [KimchiGateType.CairoClaim]: 'CairoClaim', + [KimchiGateType.CairoInstruction]: 'CairoInstruction', + [KimchiGateType.CairoFlags]: 'CairoFlags', + [KimchiGateType.CairoTransition]: 'CairoTransition', + [KimchiGateType.RangeCheck0]: 'RangeCheck0', + [KimchiGateType.RangeCheck1]: 'RangeCheck1', + [KimchiGateType.ForeignFieldAdd]: 'ForeignFieldAdd', + [KimchiGateType.ForeignFieldMul]: 'ForeignFieldMul', + [KimchiGateType.Xor16]: 'Xor16', + [KimchiGateType.Rot64]: 'Rot64', +} as const; diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts new file mode 100644 index 0000000000..4cfde3a2a3 --- /dev/null +++ b/src/lib/provable-context-debug.ts @@ -0,0 +1,175 @@ +import { FieldVector } from '../bindings/crypto/bindings/vector.js'; +import { Snarky, SnarkyState } from '../snarky.js'; +import { prettifyStacktrace } from './errors.js'; +import { FieldConst, FieldVar } from './field.js'; +import { assert } from './gadgets/common.js'; +import { + KimchiGateType, + KimchiGateTypeString, + gateTypeToString, +} from './gates.js'; +import { MlArray, MlBool, MlOption, MlString, MlTuple } from './ml/base.js'; +import { snarkContext } from './provable-context.js'; +import { assertDeepEqual } from './util/nested.js'; + +export { runCircuit, SnarkyConstraint }; + +function runCircuit( + main: () => void, + { + withWitness, + evalConstraints, + expectedConstraints, + }: { + withWitness: boolean; + evalConstraints?: boolean; + expectedConstraints?: ConstraintLog[]; + } +) { + const snarkyState = Snarky.lowLevel.state; + let numInputs = 0; + + let constraints: ConstraintLog[] = []; + + let [, state, input, aux, system] = Snarky.lowLevel.createState( + numInputs, + MlBool(evalConstraints ?? true), + MlBool(withWitness), + MlOption((_label, maybeConstraint) => { + let mlConstraint = MlOption.from(maybeConstraint); + if (mlConstraint === undefined) return; + let constraintLog = getGateTypeAndData(mlConstraint); + if (expectedConstraints !== undefined) { + let expected = expectedConstraints[constraints.length]; + assertDeepEqual(constraintLog, expected, 'constraint mismatch'); + } + constraints.push(constraintLog); + }) + ); + + let id = snarkContext.enter({ inCheckedComputation: true }); + let [, oldState] = snarkyState; + snarkyState[1] = state; + let counters = Snarky.lowLevel.pushActiveCounter(); + try { + main(); + } catch (error) { + throw prettifyStacktrace(error); + } finally { + Snarky.lowLevel.resetActiveCounter(counters); + snarkyState[1] = oldState; + snarkContext.leave(id); + } + + let witness = MlArray.mapFrom(aux, FieldConst.toBigint); + + return { constraints, witness }; +} + +type ConstraintLog = { type: ConstraintType; data: any }; + +type ConstraintType = + | KimchiGateTypeString + | 'Boolean' + | 'Equal' + | 'Square' + | 'R1CS'; + +function getGateTypeAndData(constraint: SnarkyConstraint): ConstraintLog { + let [, basic] = constraint; + switch (basic[1][1].c) { + case SnarkyConstraintType.Boolean: + return { type: 'Boolean', data: basic[2] }; + case SnarkyConstraintType.Equal: + return { type: 'Equal', data: basic[2] }; + case SnarkyConstraintType.Square: + return { type: 'Square', data: basic[2] }; + case SnarkyConstraintType.R1CS: + return { type: 'R1CS', data: basic[2] }; + case SnarkyConstraintType.Added: + // why can't TS narrow this? + let [plonkConstraint, ...data] = basic[2] as [PlonkConstraint, ...any]; + let kimchiGateType = plonkConstraintToKimchiGateType[plonkConstraint]; + assert(kimchiGateType !== undefined, 'unimplemented'); + return { type: gateTypeToString(kimchiGateType), data }; + default: + assert(false); + } +} + +type SnarkyConstraint = [ + _: 0, + basic: SnarkyConstraintBasic, + annotation: MlOption +]; + +// types that are defined by `type t = ..`, `type t += `, etc +type MlIncrementalTypeEnum = [ + 248, + { t: number; c: T; l: number }, + number +]; + +// matches Snarky_backendless.Constraint.basic +type SnarkyConstraintBasic = + | [0, MlIncrementalTypeEnum, FieldVar] + | [0, MlIncrementalTypeEnum, MlTuple] + | [ + 0, + MlIncrementalTypeEnum, + MlTuple + ] + | [0, MlIncrementalTypeEnum, MlTuple] + | [ + 0, + MlIncrementalTypeEnum<'Snarky_backendless__Constraint.Add_kind(C).T'>, + [PlonkConstraint, ...any] + ]; + +enum SnarkyConstraintType { + Boolean = 'Snarky_backendless__Constraint.Boolean', + Equal = 'Snarky_backendless__Constraint.Equal', + Square = 'Snarky_backendless__Constraint.Square', + R1CS = 'Snarky_backendless__Constraint.R1CS', + Added = 'Snarky_backendless__Constraint.Add_kind(C).T', +} + +// matches Plonk_constraint_system.Plonk_constraint.t +enum PlonkConstraint { + Basic, + Poseidon, + EC_add_complete, + EC_scale, + EC_endoscale, + EC_endoscalar, + Lookup, + RangeCheck0, + RangeCheck1, + Xor, + ForeignFieldAdd, + ForeignFieldMul, + Rot64, + AddFixedLookupTable, + AddRuntimeTableCfg, + Raw, +} + +const plonkConstraintToKimchiGateType = { + [PlonkConstraint.Basic]: KimchiGateType.Generic, + [PlonkConstraint.Poseidon]: KimchiGateType.Poseidon, + [PlonkConstraint.EC_add_complete]: KimchiGateType.CompleteAdd, + [PlonkConstraint.EC_scale]: KimchiGateType.VarBaseMul, + [PlonkConstraint.EC_endoscale]: KimchiGateType.EndoMul, + [PlonkConstraint.EC_endoscalar]: KimchiGateType.EndoMulScalar, + [PlonkConstraint.Lookup]: KimchiGateType.Lookup, + [PlonkConstraint.RangeCheck0]: KimchiGateType.RangeCheck0, + [PlonkConstraint.RangeCheck1]: KimchiGateType.RangeCheck1, + [PlonkConstraint.Xor]: KimchiGateType.Xor16, + [PlonkConstraint.ForeignFieldAdd]: KimchiGateType.ForeignFieldAdd, + [PlonkConstraint.ForeignFieldMul]: KimchiGateType.ForeignFieldMul, + [PlonkConstraint.Rot64]: KimchiGateType.Rot64, + [PlonkConstraint.AddFixedLookupTable]: undefined, + [PlonkConstraint.AddRuntimeTableCfg]: undefined, + // isn't used for other stuff + [PlonkConstraint.Raw]: KimchiGateType.Zero, +} as const; diff --git a/src/lib/provable-context-debug.unit-test.ts b/src/lib/provable-context-debug.unit-test.ts new file mode 100644 index 0000000000..ddf47a8691 --- /dev/null +++ b/src/lib/provable-context-debug.unit-test.ts @@ -0,0 +1,16 @@ +import { Field } from './core.js'; +import { Gadgets } from './gadgets/gadgets.js'; +import { runCircuit } from './provable-context-debug.js'; +import { Provable } from './provable.js'; + +let cs = runCircuit( + () => { + let x = Provable.witness(Field, () => Field(5)); + let y = x.mul(x); + Gadgets.rangeCheck16(y); + Gadgets.rangeCheck64(y); + }, + { withWitness: true } +); + +console.log(cs); diff --git a/src/lib/util/nested.ts b/src/lib/util/nested.ts new file mode 100644 index 0000000000..db49109d33 --- /dev/null +++ b/src/lib/util/nested.ts @@ -0,0 +1,44 @@ +export { assertDeepEqual, deepEqual }; + +type Nested = + | number + | bigint + | string + | boolean + | null + | undefined + | Nested[] + | { [key: string]: Nested }; + +function assertDeepEqual(a: Nested, b: Nested, message?: string) { + if (!deepEqual(a, b)) { + let fullMessage = `assertDeepEqual failed: ${message ?? ''} + +Inputs: +${JSON.stringify(a)} +${JSON.stringify(b)} +`; + throw Error(fullMessage); + } +} + +function deepEqual(a: Nested, b: Nested): boolean { + if (a === b) return true; + if (typeof a !== typeof b) return false; + if (typeof a !== 'object' || typeof b !== 'object') return false; + if (a === null || b === null) return false; + if (Array.isArray(a)) { + if (!Array.isArray(b)) return false; + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) { + if (!deepEqual(a[i], b[i])) return false; + } + return true; + } + if (Array.isArray(b)) return false; + if (Object.keys(a).length !== Object.keys(b).length) return false; + for (const key in a) { + if (!deepEqual(a[key], b[key])) return false; + } + return true; +} diff --git a/src/snarky.d.ts b/src/snarky.d.ts index 3657f3685a..7c89f5134e 100644 --- a/src/snarky.d.ts +++ b/src/snarky.d.ts @@ -26,6 +26,7 @@ import type { WasmFqSrs, } from './bindings/compiled/node_bindings/plonk_wasm.cjs'; import type { KimchiGateType } from './lib/gates.ts'; +import type { SnarkyConstraint } from './lib/provable-context-debug.js'; import type { FieldVector } from './bindings/crypto/bindings/vector.ts'; export { ProvablePure, Provable, Ledger, Pickles, Gate, GateType, getWasm }; @@ -40,7 +41,6 @@ export { FeatureFlags, MlFeatureFlags, SnarkyState, - SnarkyConstraint, }; /** @@ -590,12 +590,6 @@ type SnarkyState = [ log_constraint: unknown ]; -type SnarkyConstraint = [ - _: 0, - basic: [number, unknown], // actually this is an enum - annotation: MlOption -]; - type GateType = | 'Zero' | 'Generic' From 75a48f72c7a47772beb0782ab8d3b8e470435fe4 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 13:22:48 +0100 Subject: [PATCH 06/27] expose more flexible cs processing --- src/lib/provable-context-debug.ts | 17 +++++++----- src/lib/provable-context.ts | 45 ++++++++++++++++++++----------- src/snarky.d.ts | 14 ++++++++-- 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index 4cfde3a2a3..6e1205c985 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -1,4 +1,3 @@ -import { FieldVector } from '../bindings/crypto/bindings/vector.js'; import { Snarky, SnarkyState } from '../snarky.js'; import { prettifyStacktrace } from './errors.js'; import { FieldConst, FieldVar } from './field.js'; @@ -12,10 +11,10 @@ import { MlArray, MlBool, MlOption, MlString, MlTuple } from './ml/base.js'; import { snarkContext } from './provable-context.js'; import { assertDeepEqual } from './util/nested.js'; -export { runCircuit, SnarkyConstraint }; +export { runCircuit, SnarkyConstraint, MlConstraintSystem }; -function runCircuit( - main: () => void, +function runCircuit( + main: () => T, { withWitness, evalConstraints, @@ -28,7 +27,6 @@ function runCircuit( ) { const snarkyState = Snarky.lowLevel.state; let numInputs = 0; - let constraints: ConstraintLog[] = []; let [, state, input, aux, system] = Snarky.lowLevel.createState( @@ -51,8 +49,9 @@ function runCircuit( let [, oldState] = snarkyState; snarkyState[1] = state; let counters = Snarky.lowLevel.pushActiveCounter(); + let result: T; try { - main(); + result = main(); } catch (error) { throw prettifyStacktrace(error); } finally { @@ -63,7 +62,11 @@ function runCircuit( let witness = MlArray.mapFrom(aux, FieldConst.toBigint); - return { constraints, witness }; + return { constraints, witness, result, system }; +} + +class MlConstraintSystem { + // opaque } type ConstraintLog = { type: ConstraintType; data: any }; diff --git a/src/lib/provable-context.ts b/src/lib/provable-context.ts index c285ee81f8..bcf8b66d53 100644 --- a/src/lib/provable-context.ts +++ b/src/lib/provable-context.ts @@ -1,5 +1,11 @@ import { Context } from './global-context.js'; -import { Gate, GateType, JsonGate, Snarky } from '../snarky.js'; +import { + Gate, + GateType, + JsonConstraintSystem, + JsonGate, + Snarky, +} from '../snarky.js'; import { parseHexString32 } from '../bindings/crypto/bigint-helpers.js'; import { prettifyStacktrace } from './errors.js'; import { Fp } from '../bindings/crypto/finite_field.js'; @@ -19,6 +25,7 @@ export { inCompileMode, gatesFromJson, printGates, + constraintSystemFromJson, }; // global circuit-related context @@ -95,25 +102,11 @@ function constraintSystem(f: () => T) { let { rows, digest, json } = Snarky.run.constraintSystem(() => { result = f(); }); - let { gates, publicInputSize } = gatesFromJson(json); return { rows, digest, result: result! as T, - gates, - publicInputSize, - print() { - printGates(gates); - }, - summary() { - let gateTypes: Partial> = {}; - gateTypes['Total rows'] = rows; - for (let gate of gates) { - gateTypes[gate.type] ??= 0; - gateTypes[gate.type]!++; - } - return gateTypes; - }, + ...constraintSystemFromJson(json), }; } catch (error) { throw prettifyStacktrace(error); @@ -122,6 +115,26 @@ function constraintSystem(f: () => T) { } } +function constraintSystemFromJson(json: JsonConstraintSystem) { + let { gates, publicInputSize } = gatesFromJson(json); + return { + gates, + publicInputSize, + print() { + printGates(gates); + }, + summary() { + let gateTypes: Partial> = {}; + gateTypes['Total rows'] = gates.length; + for (let gate of gates) { + gateTypes[gate.type] ??= 0; + gateTypes[gate.type]!++; + } + return gateTypes; + }, + }; +} + // helpers function gatesFromJson(cs: { gates: JsonGate[]; public_input_size: number }) { diff --git a/src/snarky.d.ts b/src/snarky.d.ts index 7c89f5134e..8076b94a58 100644 --- a/src/snarky.d.ts +++ b/src/snarky.d.ts @@ -26,7 +26,10 @@ import type { WasmFqSrs, } from './bindings/compiled/node_bindings/plonk_wasm.cjs'; import type { KimchiGateType } from './lib/gates.ts'; -import type { SnarkyConstraint } from './lib/provable-context-debug.js'; +import type { + SnarkyConstraint, + MlConstraintSystem, +} from './lib/provable-context-debug.js'; import type { FieldVector } from './bindings/crypto/bindings/vector.ts'; export { ProvablePure, Provable, Ledger, Pickles, Gate, GateType, getWasm }; @@ -41,6 +44,7 @@ export { FeatureFlags, MlFeatureFlags, SnarkyState, + JsonConstraintSystem, }; /** @@ -561,11 +565,17 @@ declare const Snarky: { state: SnarkyState, input: FieldVector, aux: FieldVector, - system: ConstraintSystem + system: MlConstraintSystem ]; pushActiveCounter(): MlList; resetActiveCounter(counters: MlList): void; + + constraintSystem: { + getRows(system: MlConstraintSystem): number; + digest(system: MlConstraintSystem): string; + toJson(system: MlConstraintSystem): JsonConstraintSystem; + }; }; }; From cbf40cdfb0dc9d8fd5665f452eac10f4c3b7e246 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 13:59:54 +0100 Subject: [PATCH 07/27] revert returning result --- src/lib/provable-context-debug.ts | 16 ++++++++-------- src/lib/provable-context-debug.unit-test.ts | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index 6e1205c985..0543ecc671 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -1,4 +1,4 @@ -import { Snarky, SnarkyState } from '../snarky.js'; +import { Snarky } from '../snarky.js'; import { prettifyStacktrace } from './errors.js'; import { FieldConst, FieldVar } from './field.js'; import { assert } from './gadgets/common.js'; @@ -11,10 +11,10 @@ import { MlArray, MlBool, MlOption, MlString, MlTuple } from './ml/base.js'; import { snarkContext } from './provable-context.js'; import { assertDeepEqual } from './util/nested.js'; -export { runCircuit, SnarkyConstraint, MlConstraintSystem }; +export { runCircuit, SnarkyConstraint, ConstraintLog, MlConstraintSystem }; -function runCircuit( - main: () => T, +function runCircuit( + main: () => void, { withWitness, evalConstraints, @@ -45,13 +45,12 @@ function runCircuit( }) ); - let id = snarkContext.enter({ inCheckedComputation: true }); + let id = snarkContext.enter({ inAnalyze: true, inCheckedComputation: true }); let [, oldState] = snarkyState; snarkyState[1] = state; let counters = Snarky.lowLevel.pushActiveCounter(); - let result: T; try { - result = main(); + main(); } catch (error) { throw prettifyStacktrace(error); } finally { @@ -60,9 +59,10 @@ function runCircuit( snarkContext.leave(id); } + let publicInput = MlArray.mapFrom(input, FieldConst.toBigint); let witness = MlArray.mapFrom(aux, FieldConst.toBigint); - return { constraints, witness, result, system }; + return { constraints, publicInput, witness, system }; } class MlConstraintSystem { diff --git a/src/lib/provable-context-debug.unit-test.ts b/src/lib/provable-context-debug.unit-test.ts index ddf47a8691..1ffbbd3c77 100644 --- a/src/lib/provable-context-debug.unit-test.ts +++ b/src/lib/provable-context-debug.unit-test.ts @@ -10,7 +10,7 @@ let cs = runCircuit( Gadgets.rangeCheck16(y); Gadgets.rangeCheck64(y); }, - { withWitness: true } + { withWitness: false } ); console.log(cs); From 02fdbc871bb9fc0220348e7bc5836ec3dea9dc8e Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 14:01:00 +0100 Subject: [PATCH 08/27] don't return result from constraint system --- src/lib/provable-context.ts | 8 ++------ src/lib/zkapp.ts | 7 ++++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/lib/provable-context.ts b/src/lib/provable-context.ts index bcf8b66d53..224e67cec8 100644 --- a/src/lib/provable-context.ts +++ b/src/lib/provable-context.ts @@ -95,17 +95,13 @@ function runUnchecked(f: () => void) { } } -function constraintSystem(f: () => T) { +function constraintSystem(f: () => void) { let id = snarkContext.enter({ inAnalyze: true, inCheckedComputation: true }); try { - let result: T; - let { rows, digest, json } = Snarky.run.constraintSystem(() => { - result = f(); - }); + let { rows, digest, json } = Snarky.run.constraintSystem(f); return { rows, digest, - result: result! as T, ...constraintSystemFromJson(json), }; } catch (error) { diff --git a/src/lib/zkapp.ts b/src/lib/zkapp.ts index 9346973f72..d37cc7d96c 100644 --- a/src/lib/zkapp.ts +++ b/src/lib/zkapp.ts @@ -1186,7 +1186,8 @@ super.init(); try { for (let methodIntf of methodIntfs) { let accountUpdate: AccountUpdate; - let { rows, digest, result, gates } = analyzeMethod( + let hasReturn = false; + let { rows, digest, gates } = analyzeMethod( ZkappPublicInput, methodIntf, (publicInput, publicKey, tokenId, ...args) => { @@ -1195,15 +1196,15 @@ super.init(); publicInput, ...args ); + hasReturn = result !== undefined; accountUpdate = instance.#executionState!.accountUpdate; - return result; } ); methodMetadata[methodIntf.methodName] = { actions: accountUpdate!.body.actions.data.length, rows, digest, - hasReturn: result !== undefined, + hasReturn, gates, }; } From 08f9f7244b1bb083f1fa93d1f3227b027bd340fb Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 14:02:05 +0100 Subject: [PATCH 09/27] use runCircuit in analyzeMethods --- src/lib/proof_system.ts | 27 +++++++++++++++++++-------- src/lib/zkapp.ts | 5 ++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/lib/proof_system.ts b/src/lib/proof_system.ts index 59ebf57ec4..420ec8d917 100644 --- a/src/lib/proof_system.ts +++ b/src/lib/proof_system.ts @@ -11,6 +11,7 @@ import { MlFeatureFlags, Gate, GateType, + Snarky, } from '../snarky.js'; import { Field, Bool } from './core.js'; import { @@ -25,7 +26,7 @@ import { } from './circuit_value.js'; import { Provable } from './provable.js'; import { assert, prettifyStacktracePromise } from './errors.js'; -import { snarkContext } from './provable-context.js'; +import { constraintSystemFromJson, snarkContext } from './provable-context.js'; import { hashConstant } from './hash.js'; import { MlArray, MlBool, MlResult, MlPair } from './ml/base.js'; import { MlFieldArray, MlFieldConstArray } from './ml/fields.js'; @@ -37,6 +38,7 @@ import { parseHeader, } from './proof-system/prover-keys.js'; import { setSrsCache, unsetSrsCache } from '../bindings/crypto/bindings/srs.js'; +import { runCircuit } from './provable-context-debug.js'; // public API export { @@ -731,13 +733,22 @@ function analyzeMethod( methodIntf: MethodInterface, method: (...args: any) => T ) { - return Provable.constraintSystem(() => { - let args = synthesizeMethodArguments(methodIntf, true); - let publicInput = emptyWitness(publicInputType); - if (publicInputType === Undefined || publicInputType === Void) - return method(...args); - return method(publicInput, ...args); - }); + let { constraints, system } = runCircuit( + () => { + let args = synthesizeMethodArguments(methodIntf, true); + let publicInput = emptyWitness(publicInputType); + if (publicInputType === Undefined || publicInputType === Void) { + method(...args); + } else { + method(publicInput, ...args); + } + }, + { withWitness: false } + ); + let digest = Snarky.lowLevel.constraintSystem.digest(system); + let csJson = Snarky.lowLevel.constraintSystem.toJson(system); + let cs = constraintSystemFromJson(csJson); + return { constraints, rows: cs.gates.length, digest, ...cs }; } function picklesRuleFromFunction( diff --git a/src/lib/zkapp.ts b/src/lib/zkapp.ts index d37cc7d96c..fe88881f1d 100644 --- a/src/lib/zkapp.ts +++ b/src/lib/zkapp.ts @@ -56,6 +56,7 @@ import { snarkContext, } from './provable-context.js'; import { Cache } from './proof-system/cache.js'; +import type { ConstraintLog } from './provable-context-debug.js'; // external API export { @@ -616,6 +617,7 @@ class SmartContract { digest: string; hasReturn: boolean; gates: Gate[]; + expectedConstraints?: ConstraintLog[]; } >; // keyed by method name static _provers?: Pickles.Prover[]; @@ -1187,7 +1189,7 @@ super.init(); for (let methodIntf of methodIntfs) { let accountUpdate: AccountUpdate; let hasReturn = false; - let { rows, digest, gates } = analyzeMethod( + let { rows, digest, gates, constraints } = analyzeMethod( ZkappPublicInput, methodIntf, (publicInput, publicKey, tokenId, ...args) => { @@ -1206,6 +1208,7 @@ super.init(); digest, hasReturn, gates, + expectedConstraints: constraints, }; } } finally { From b3a3479726ff34d882e5201a1a7164cbe6d8a4ef Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 15:07:25 +0100 Subject: [PATCH 10/27] move zkapp proving logic to separate file --- src/lib/account_update.ts | 133 +------------------------------ src/lib/mina.ts | 2 +- src/lib/mina/zkapp-proof.ts | 152 ++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 131 deletions(-) create mode 100644 src/lib/mina/zkapp-proof.ts diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index 23f5e66c08..c1884352f0 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -47,7 +47,6 @@ export { FeePayerUnsigned, ZkappCommand, addMissingSignatures, - addMissingProofs, ZkappStateLength, Events, Actions, @@ -57,8 +56,11 @@ export { createChildAccountUpdate, AccountUpdatesLayout, zkAppProver, + ZkappProverData, SmartContractContext, dummySignature, + LazySignature, + LazyProof, }; const ZkappStateLength = 8; @@ -1748,11 +1750,6 @@ type ZkappCommandSigned = { accountUpdates: (AccountUpdate & { lazyAuthorization?: LazyProof })[]; memo: string; }; -type ZkappCommandProved = { - feePayer: FeePayerUnsigned; - accountUpdates: (AccountUpdate & { lazyAuthorization?: LazySignature })[]; - memo: string; -}; const ZkappCommand = { toPretty(transaction: ZkappCommand) { @@ -1976,127 +1973,3 @@ type ZkappPublicInput = { calls: Field; }; let ZkappPublicInput = provablePure({ accountUpdate: Field, calls: Field }); - -async function addMissingProofs( - zkappCommand: ZkappCommand, - { proofsEnabled = true } -): Promise<{ - zkappCommand: ZkappCommandProved; - proofs: (Proof | undefined)[]; -}> { - let { feePayer, accountUpdates, memo } = zkappCommand; - // compute proofs serially. in parallel would clash with our global variable - // hacks - let accountUpdatesProved: AccountUpdateProved[] = []; - let proofs: (Proof | undefined)[] = []; - for (let i = 0; i < accountUpdates.length; i++) { - let { accountUpdateProved, proof } = await addProof( - zkappCommand, - i, - proofsEnabled - ); - accountUpdatesProved.push(accountUpdateProved); - proofs.push(proof); - } - return { - zkappCommand: { feePayer, accountUpdates: accountUpdatesProved, memo }, - proofs, - }; -} - -async function addProof( - transaction: ZkappCommand, - index: number, - proofsEnabled: boolean -) { - let accountUpdate = transaction.accountUpdates[index]; - accountUpdate = AccountUpdate.clone(accountUpdate); - - if (accountUpdate.lazyAuthorization?.kind !== 'lazy-proof') { - return { - accountUpdateProved: accountUpdate as AccountUpdateProved, - proof: undefined, - }; - } - if (!proofsEnabled) { - Authorization.setProof(accountUpdate, await dummyBase64Proof()); - return { - accountUpdateProved: accountUpdate as AccountUpdateProved, - proof: undefined, - }; - } - - let lazyProof: LazyProof = accountUpdate.lazyAuthorization; - let prover = getZkappProver(lazyProof); - let proverData = { transaction, accountUpdate, index }; - let proof = await createZkappProof(prover, lazyProof, proverData); - - let accountUpdateProved = Authorization.setProof( - accountUpdate, - Pickles.proofToBase64Transaction(proof.proof) - ); - return { accountUpdateProved, proof }; -} - -async function createZkappProof( - prover: Pickles.Prover, - { - methodName, - args, - previousProofs, - ZkappClass, - memoized, - blindingValue, - }: LazyProof, - { transaction, accountUpdate, index }: ZkappProverData -): Promise> { - let publicInput = accountUpdate.toPublicInput(); - let publicInputFields = MlFieldConstArray.to( - ZkappPublicInput.toFields(publicInput) - ); - - let [, , proof] = await zkAppProver.run( - [accountUpdate.publicKey, accountUpdate.tokenId, ...args], - { transaction, accountUpdate, index }, - async () => { - let id = memoizationContext.enter({ - memoized, - currentIndex: 0, - blindingValue, - }); - try { - return await prover(publicInputFields, MlArray.to(previousProofs)); - } catch (err) { - console.error(`Error when proving ${ZkappClass.name}.${methodName}()`); - throw err; - } finally { - memoizationContext.leave(id); - } - } - ); - - let maxProofsVerified = ZkappClass._maxProofsVerified!; - const Proof = ZkappClass.Proof(); - return new Proof({ - publicInput, - publicOutput: undefined, - proof, - maxProofsVerified, - }); -} - -function getZkappProver({ methodName, ZkappClass }: LazyProof) { - if (ZkappClass._provers === undefined) - throw Error( - `Cannot prove execution of ${methodName}(), no prover found. ` + - `Try calling \`await ${ZkappClass.name}.compile()\` first, this will cache provers in the background.` - ); - let provers = ZkappClass._provers; - let methodError = - `Error when computing proofs: Method ${methodName} not found. ` + - `Make sure your environment supports decorators, and annotate with \`@method ${methodName}\`.`; - if (ZkappClass._methods === undefined) throw Error(methodError); - let i = ZkappClass._methods.findIndex((m) => m.methodName === methodName); - if (i === -1) throw Error(methodError); - return provers[i]; -} diff --git a/src/lib/mina.ts b/src/lib/mina.ts index e3b5823a9c..07c38282e6 100644 --- a/src/lib/mina.ts +++ b/src/lib/mina.ts @@ -3,7 +3,6 @@ import { Field } from './core.js'; import { UInt32, UInt64 } from './int.js'; import { PrivateKey, PublicKey } from './signature.js'; import { - addMissingProofs, addMissingSignatures, FeePayerUnsigned, ZkappCommand, @@ -33,6 +32,7 @@ import { transactionCommitments, verifyAccountUpdateSignature, } from '../mina-signer/src/sign-zkapp-command.js'; +import { addMissingProofs } from './mina/zkapp-proof.js'; export { createTransaction, diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts new file mode 100644 index 0000000000..1e4848a640 --- /dev/null +++ b/src/lib/mina/zkapp-proof.ts @@ -0,0 +1,152 @@ +import { Pickles } from '../../snarky.js'; +import { + AccountUpdate, + Authorization, + FeePayerUnsigned, + LazyProof, + LazySignature, + ZkappCommand, + ZkappProverData, + ZkappPublicInput, + zkAppProver, +} from '../account_update.js'; +import { MlArray } from '../ml/base.js'; +import { MlFieldConstArray } from '../ml/fields.js'; +import { Empty, Proof, dummyBase64Proof } from '../proof_system.js'; +import { memoizationContext } from '../provable.js'; + +export { addMissingProofs }; + +type AccountUpdateProved = AccountUpdate & { + lazyAuthorization?: LazySignature; +}; + +type ZkappCommandProved = { + feePayer: FeePayerUnsigned; + accountUpdates: AccountUpdateProved[]; + memo: string; +}; + +async function addMissingProofs( + zkappCommand: ZkappCommand, + { proofsEnabled = true } +): Promise<{ + zkappCommand: ZkappCommandProved; + proofs: (Proof | undefined)[]; +}> { + let { feePayer, accountUpdates, memo } = zkappCommand; + // compute proofs serially. in parallel would clash with our global variable + // hacks + let accountUpdatesProved: AccountUpdateProved[] = []; + let proofs: (Proof | undefined)[] = []; + for (let i = 0; i < accountUpdates.length; i++) { + let { accountUpdateProved, proof } = await addProof( + zkappCommand, + i, + proofsEnabled + ); + accountUpdatesProved.push(accountUpdateProved); + proofs.push(proof); + } + return { + zkappCommand: { feePayer, accountUpdates: accountUpdatesProved, memo }, + proofs, + }; +} + +async function addProof( + transaction: ZkappCommand, + index: number, + proofsEnabled: boolean +) { + let accountUpdate = transaction.accountUpdates[index]; + accountUpdate = AccountUpdate.clone(accountUpdate); + + if (accountUpdate.lazyAuthorization?.kind !== 'lazy-proof') { + return { + accountUpdateProved: accountUpdate as AccountUpdateProved, + proof: undefined, + }; + } + if (!proofsEnabled) { + Authorization.setProof(accountUpdate, await dummyBase64Proof()); + return { + accountUpdateProved: accountUpdate as AccountUpdateProved, + proof: undefined, + }; + } + + let lazyProof: LazyProof = accountUpdate.lazyAuthorization; + let prover = getZkappProver(lazyProof); + let proverData = { transaction, accountUpdate, index }; + let proof = await createZkappProof(prover, lazyProof, proverData); + + let accountUpdateProved = Authorization.setProof( + accountUpdate, + Pickles.proofToBase64Transaction(proof.proof) + ); + return { accountUpdateProved, proof }; +} + +async function createZkappProof( + prover: Pickles.Prover, + { + methodName, + args, + previousProofs, + ZkappClass, + memoized, + blindingValue, + }: LazyProof, + { transaction, accountUpdate, index }: ZkappProverData +): Promise> { + let publicInput = accountUpdate.toPublicInput(); + let publicInputFields = MlFieldConstArray.to( + ZkappPublicInput.toFields(publicInput) + ); + + let [, , proof] = await zkAppProver.run( + [accountUpdate.publicKey, accountUpdate.tokenId, ...args], + { transaction, accountUpdate, index }, + async () => { + let id = memoizationContext.enter({ + memoized, + currentIndex: 0, + blindingValue, + }); + try { + return await prover(publicInputFields, MlArray.to(previousProofs)); + } catch (err) { + console.error(`Error when proving ${ZkappClass.name}.${methodName}()`); + throw err; + } finally { + memoizationContext.leave(id); + } + } + ); + + let maxProofsVerified = ZkappClass._maxProofsVerified!; + const Proof = ZkappClass.Proof(); + return new Proof({ + publicInput, + publicOutput: undefined, + proof, + maxProofsVerified, + }); +} + +function getZkappProver({ methodName, ZkappClass }: LazyProof) { + if (ZkappClass._provers === undefined) + throw Error( + `Cannot prove execution of ${methodName}(), no prover found. ` + + `Try calling \`await ${ZkappClass.name}.compile()\` first, this will cache provers in the background.` + ); + let provers = ZkappClass._provers; + let methodError = + `Error when computing proofs: Method ${methodName} not found. ` + + `Make sure your environment supports decorators, and annotate with \`@method ${methodName}\`.`; + if (ZkappClass._methods === undefined) throw Error(methodError); + let i = ZkappClass._methods.findIndex((m) => m.methodName === methodName); + if (i === -1) throw Error(methodError); + return provers[i]; +} From 089e19eb372c23f702e733ad53ccd98727fe4cce Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 15:10:20 +0100 Subject: [PATCH 11/27] minor --- src/lib/account_update.ts | 40 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index c1884352f0..db5abf8d7b 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -4,7 +4,7 @@ import { provable, provablePure, } from './circuit_value.js'; -import { memoizationContext, memoizeWitness, Provable } from './provable.js'; +import { memoizeWitness, Provable } from './provable.js'; import { Field, Bool } from './core.js'; import { Pickles, Test } from '../snarky.js'; import { jsLayout } from '../bindings/mina-transaction/gen/js-layout.js'; @@ -18,7 +18,7 @@ import { UInt64, UInt32, Int64, Sign } from './int.js'; import * as Mina from './mina.js'; import { SmartContract } from './zkapp.js'; import * as Precondition from './precondition.js'; -import { dummyBase64Proof, Empty, Proof, Prover } from './proof_system.js'; +import { Proof, Prover } from './proof_system.js'; import { Memo } from '../mina-signer/src/memo.js'; import { Events, @@ -29,9 +29,7 @@ import { hashWithPrefix, packToFields } from './hash.js'; import { mocks, prefixes } from '../bindings/crypto/constants.js'; import { Context } from './global-context.js'; import { assert } from './errors.js'; -import { MlArray } from './ml/base.js'; import { Signature, signFieldElement } from '../mina-signer/src/signature.js'; -import { MlFieldConstArray } from './ml/fields.js'; import { transactionCommitments } from '../mina-signer/src/sign-zkapp-command.js'; // external API @@ -98,8 +96,8 @@ type Preconditions = AccountUpdateBody['preconditions']; */ type SetOrKeep = { isSome: Bool; value: T }; -const True = () => Bool(true); -const False = () => Bool(false); +const True = Bool(true); +const False = Bool(false); /** * One specific permission value. @@ -116,45 +114,45 @@ let Permission = { * Modification is impossible. */ impossible: (): Permission => ({ - constant: True(), - signatureNecessary: True(), - signatureSufficient: False(), + constant: True, + signatureNecessary: True, + signatureSufficient: False, }), /** * Modification is always permitted */ none: (): Permission => ({ - constant: True(), - signatureNecessary: False(), - signatureSufficient: True(), + constant: True, + signatureNecessary: False, + signatureSufficient: True, }), /** * Modification is permitted by zkapp proofs only */ proof: (): Permission => ({ - constant: False(), - signatureNecessary: False(), - signatureSufficient: False(), + constant: False, + signatureNecessary: False, + signatureSufficient: False, }), /** * Modification is permitted by signatures only, using the private key of the zkapp account */ signature: (): Permission => ({ - constant: False(), - signatureNecessary: True(), - signatureSufficient: True(), + constant: False, + signatureNecessary: True, + signatureSufficient: True, }), /** * Modification is permitted by zkapp proofs or signatures */ proofOrSignature: (): Permission => ({ - constant: False(), - signatureNecessary: False(), - signatureSufficient: True(), + constant: False, + signatureNecessary: False, + signatureSufficient: True, }), }; From 5121c46c5ff9b6b5492662631969ee69763ae7d9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 17:12:55 +0100 Subject: [PATCH 12/27] WIP: compare constraints in zkapp proof to the ones created in analyzeMethods --- src/index.ts | 3 ++ src/lib/mina/zkapp-proof.ts | 58 +++++++++++++++++++++++++++++-- src/lib/proof_system.ts | 43 ++++++++++++++++++++++- src/lib/provable-context-debug.ts | 27 ++++++++++---- src/lib/util/nested.ts | 11 +++--- 5 files changed, 128 insertions(+), 14 deletions(-) diff --git a/src/index.ts b/src/index.ts index 243fe7dbfa..378d7a9c7c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,6 +48,9 @@ export { } from './lib/zkapp.js'; export { state, State, declareState } from './lib/state.js'; +// TODO expose this in a cleaner way +export { runAsIfProver } from './lib/mina/zkapp-proof.js'; + export type { JsonProof } from './lib/proof_system.js'; export { Proof, diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts index 1e4848a640..22668e6f14 100644 --- a/src/lib/mina/zkapp-proof.ts +++ b/src/lib/mina/zkapp-proof.ts @@ -10,12 +10,19 @@ import { ZkappPublicInput, zkAppProver, } from '../account_update.js'; +import { assert } from '../gadgets/common.js'; import { MlArray } from '../ml/base.js'; import { MlFieldConstArray } from '../ml/fields.js'; -import { Empty, Proof, dummyBase64Proof } from '../proof_system.js'; -import { memoizationContext } from '../provable.js'; +import { + Empty, + Proof, + dummyBase64Proof, + methodArgumentsToVars, +} from '../proof_system.js'; +import { runCircuit } from '../provable-context-debug.js'; +import { Provable, memoizationContext } from '../provable.js'; -export { addMissingProofs }; +export { addMissingProofs, runAsIfProver }; type AccountUpdateProved = AccountUpdate & { lazyAuthorization?: LazySignature; @@ -150,3 +157,48 @@ function getZkappProver({ methodName, ZkappClass }: LazyProof) { if (i === -1) throw Error(methodError); return provers[i]; } + +// for debugging prove/compile discrepancies +// TODO run this automatically when detecting some problem + +function runAsIfProver(transaction: ZkappCommand, index: number) { + let accountUpdate = transaction.accountUpdates[index]; + accountUpdate = AccountUpdate.clone(accountUpdate); + + assert( + accountUpdate.lazyAuthorization?.kind === 'lazy-proof', + 'Account update is not associated with a provable method call' + ); + + let { methodName, ZkappClass, args } = accountUpdate.lazyAuthorization; + let metadata = ZkappClass._methodMetadata?.[methodName]; + let methodIntf = ZkappClass._methods?.find( + (m) => m.methodName === methodName + ); + + assert( + metadata !== undefined && methodIntf !== undefined, + `No metadata found for zkapp method ${methodName}()` + ); + + let publicInput = accountUpdate.toPublicInput(); + let proverData = { transaction, accountUpdate, index }; + + runCircuit( + () => { + let [pk, tid, ...otherArgs] = methodArgumentsToVars( + [accountUpdate.publicKey, accountUpdate.tokenId, ...args], + methodIntf! + ).args; + publicInput = Provable.witness(ZkappPublicInput, () => publicInput); + + let instance = new ZkappClass(pk, tid); + (instance as any)[methodName](publicInput, ...otherArgs); + }, + { + withWitness: true, + snarkContext: { proverData, inAnalyze: true }, + expectedConstraints: metadata.expectedConstraints, + } + ); +} diff --git a/src/lib/proof_system.ts b/src/lib/proof_system.ts index 420ec8d917..ce65ae23bb 100644 --- a/src/lib/proof_system.ts +++ b/src/lib/proof_system.ts @@ -66,6 +66,7 @@ export { analyzeMethod, emptyValue, emptyWitness, + methodArgumentsToVars, synthesizeMethodArguments, methodArgumentsToConstant, methodArgumentTypesAndValues, @@ -743,7 +744,7 @@ function analyzeMethod( method(publicInput, ...args); } }, - { withWitness: false } + { withWitness: false, snarkContext: { inAnalyze: true } } ); let digest = Snarky.lowLevel.constraintSystem.digest(system); let csJson = Snarky.lowLevel.constraintSystem.toJson(system); @@ -843,6 +844,46 @@ function picklesRuleFromFunction( }; } +// TODO share logic with picklesRuleFromFunction + +function methodArgumentsToVars( + argsWithoutPublicInput: any[], + { allArgs, proofArgs, witnessArgs }: MethodInterface +) { + let finalArgs = []; + let proofs: Proof[] = []; + let previousStatements: Pickles.Statement[] = []; + for (let i = 0; i < allArgs.length; i++) { + let arg = allArgs[i]; + if (arg.type === 'witness') { + let type = witnessArgs[arg.index]; + finalArgs[i] = Provable.witness(type, () => { + return argsWithoutPublicInput?.[i] ?? emptyValue(type); + }); + } else if (arg.type === 'proof') { + let Proof = proofArgs[arg.index]; + let type = getStatementType(Proof); + let proof_ = (argsWithoutPublicInput?.[i] as Proof) ?? { + proof: undefined, + publicInput: emptyValue(type.input), + publicOutput: emptyValue(type.output), + }; + let { proof, publicInput, publicOutput } = proof_; + publicInput = Provable.witness(type.input, () => publicInput); + publicOutput = Provable.witness(type.output, () => publicOutput); + let proofInstance = new Proof({ publicInput, publicOutput, proof }); + finalArgs[i] = proofInstance; + proofs.push(proofInstance); + let input = toFieldVars(type.input, publicInput); + let output = toFieldVars(type.output, publicOutput); + previousStatements.push(MlPair(input, output)); + } else if (arg.type === 'generic') { + finalArgs[i] = argsWithoutPublicInput?.[i] ?? emptyGeneric(); + } + } + return { args: finalArgs, proofs, previousStatements }; +} + function synthesizeMethodArguments( { allArgs, proofArgs, witnessArgs }: MethodInterface, asVariables = false diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index 0543ecc671..a9897cc481 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -8,8 +8,8 @@ import { gateTypeToString, } from './gates.js'; import { MlArray, MlBool, MlOption, MlString, MlTuple } from './ml/base.js'; -import { snarkContext } from './provable-context.js'; -import { assertDeepEqual } from './util/nested.js'; +import { SnarkContext, snarkContext } from './provable-context.js'; +import { assertDeepEqual, deepEqual } from './util/nested.js'; export { runCircuit, SnarkyConstraint, ConstraintLog, MlConstraintSystem }; @@ -19,10 +19,12 @@ function runCircuit( withWitness, evalConstraints, expectedConstraints, + snarkContext: ctx = {}, }: { withWitness: boolean; evalConstraints?: boolean; expectedConstraints?: ConstraintLog[]; + snarkContext?: SnarkContext; } ) { const snarkyState = Snarky.lowLevel.state; @@ -37,15 +39,28 @@ function runCircuit( let mlConstraint = MlOption.from(maybeConstraint); if (mlConstraint === undefined) return; let constraintLog = getGateTypeAndData(mlConstraint); + constraints.push(constraintLog); + + // TODO remove + if (constraints.length < 5) + console.log(prettifyStacktrace(Error(constraintLog.type))); + // console.log(constraintLog); if (expectedConstraints !== undefined) { - let expected = expectedConstraints[constraints.length]; - assertDeepEqual(constraintLog, expected, 'constraint mismatch'); + let expected = expectedConstraints[constraints.length - 1]; + // assertDeepEqual(constraintLog, expected, 'constraint mismatch'); + if (!deepEqual(constraintLog, expected)) { + console.log('actual', constraints); + console.log( + 'expected', + expectedConstraints.slice(0, constraints.length) + ); + throw Error('constraint mismatch'); + } } - constraints.push(constraintLog); }) ); - let id = snarkContext.enter({ inAnalyze: true, inCheckedComputation: true }); + let id = snarkContext.enter({ inCheckedComputation: true, ...ctx }); let [, oldState] = snarkyState; snarkyState[1] = state; let counters = Snarky.lowLevel.pushActiveCounter(); diff --git a/src/lib/util/nested.ts b/src/lib/util/nested.ts index db49109d33..bee161d8fd 100644 --- a/src/lib/util/nested.ts +++ b/src/lib/util/nested.ts @@ -10,13 +10,16 @@ type Nested = | Nested[] | { [key: string]: Nested }; -function assertDeepEqual(a: Nested, b: Nested, message?: string) { - if (!deepEqual(a, b)) { +function assertDeepEqual(actual: Nested, expected: Nested, message?: string) { + if (!deepEqual(actual, expected)) { + (BigInt.prototype as any).toJSON = function () { + return this.toString(); + }; let fullMessage = `assertDeepEqual failed: ${message ?? ''} Inputs: -${JSON.stringify(a)} -${JSON.stringify(b)} +actual: ${JSON.stringify(actual)} +expected: ${JSON.stringify(expected)} `; throw Error(fullMessage); } From 7b58e972b87395b9e55c4d491293df9d9fdb403c Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 21:21:30 +0100 Subject: [PATCH 13/27] remove debugging, fix basic constraint data --- src/lib/provable-context-debug.ts | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index a9897cc481..88d8a0ab36 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -41,21 +41,9 @@ function runCircuit( let constraintLog = getGateTypeAndData(mlConstraint); constraints.push(constraintLog); - // TODO remove - if (constraints.length < 5) - console.log(prettifyStacktrace(Error(constraintLog.type))); - // console.log(constraintLog); if (expectedConstraints !== undefined) { let expected = expectedConstraints[constraints.length - 1]; - // assertDeepEqual(constraintLog, expected, 'constraint mismatch'); - if (!deepEqual(constraintLog, expected)) { - console.log('actual', constraints); - console.log( - 'expected', - expectedConstraints.slice(0, constraints.length) - ); - throw Error('constraint mismatch'); - } + assertDeepEqual(constraintLog, expected, 'constraint mismatch'); } }) ); @@ -97,13 +85,13 @@ function getGateTypeAndData(constraint: SnarkyConstraint): ConstraintLog { let [, basic] = constraint; switch (basic[1][1].c) { case SnarkyConstraintType.Boolean: - return { type: 'Boolean', data: basic[2] }; + return { type: 'Boolean', data: basic.slice(2) }; case SnarkyConstraintType.Equal: - return { type: 'Equal', data: basic[2] }; + return { type: 'Equal', data: basic.slice(2) }; case SnarkyConstraintType.Square: - return { type: 'Square', data: basic[2] }; + return { type: 'Square', data: basic.slice(2) }; case SnarkyConstraintType.R1CS: - return { type: 'R1CS', data: basic[2] }; + return { type: 'R1CS', data: basic.slice(2) }; case SnarkyConstraintType.Added: // why can't TS narrow this? let [plonkConstraint, ...data] = basic[2] as [PlonkConstraint, ...any]; From a2064f62655e323aef912607e0077586347dcc96 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 21:22:03 +0100 Subject: [PATCH 14/27] add memoized witnesses to avoid failing constraints --- src/lib/mina/zkapp-proof.ts | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts index 22668e6f14..c93daa72c7 100644 --- a/src/lib/mina/zkapp-proof.ts +++ b/src/lib/mina/zkapp-proof.ts @@ -170,7 +170,8 @@ function runAsIfProver(transaction: ZkappCommand, index: number) { 'Account update is not associated with a provable method call' ); - let { methodName, ZkappClass, args } = accountUpdate.lazyAuthorization; + let { methodName, ZkappClass, args, memoized, blindingValue } = + accountUpdate.lazyAuthorization; let metadata = ZkappClass._methodMetadata?.[methodName]; let methodIntf = ZkappClass._methods?.find( (m) => m.methodName === methodName @@ -186,14 +187,23 @@ function runAsIfProver(transaction: ZkappCommand, index: number) { runCircuit( () => { - let [pk, tid, ...otherArgs] = methodArgumentsToVars( - [accountUpdate.publicKey, accountUpdate.tokenId, ...args], - methodIntf! - ).args; - publicInput = Provable.witness(ZkappPublicInput, () => publicInput); - - let instance = new ZkappClass(pk, tid); - (instance as any)[methodName](publicInput, ...otherArgs); + let id = memoizationContext.enter({ + memoized, + currentIndex: 0, + blindingValue, + }); + try { + let [pk, tid, ...otherArgs] = methodArgumentsToVars( + [accountUpdate.publicKey, accountUpdate.tokenId, ...args], + methodIntf! + ).args; + publicInput = Provable.witness(ZkappPublicInput, () => publicInput); + + let instance = new ZkappClass(pk, tid); + (instance as any)[methodName](publicInput, ...otherArgs); + } finally { + memoizationContext.leave(id); + } }, { withWitness: true, From 923c5ac72a417a7c2d3387fd09a5dcd517e00d19 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 10 Jan 2024 22:10:19 +0100 Subject: [PATCH 15/27] hook prover consistency check into prover when detecting a relevant problem --- src/index.ts | 3 --- src/lib/mina/zkapp-proof.ts | 18 ++++++++++++++++-- src/lib/provable-context-debug.ts | 13 ++++++++++--- src/lib/util/nested.ts | 5 ++--- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/index.ts b/src/index.ts index 378d7a9c7c..243fe7dbfa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -48,9 +48,6 @@ export { } from './lib/zkapp.js'; export { state, State, declareState } from './lib/state.js'; -// TODO expose this in a cleaner way -export { runAsIfProver } from './lib/mina/zkapp-proof.js'; - export type { JsonProof } from './lib/proof_system.js'; export { Proof, diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts index c93daa72c7..8f573bcd24 100644 --- a/src/lib/mina/zkapp-proof.ts +++ b/src/lib/mina/zkapp-proof.ts @@ -124,7 +124,19 @@ async function createZkappProof( try { return await prover(publicInputFields, MlArray.to(previousProofs)); } catch (err) { - console.error(`Error when proving ${ZkappClass.name}.${methodName}()`); + console.error( + `\n\nError when proving ${ZkappClass.name}.${methodName}()` + ); + if ( + err instanceof Error && + err.message.includes('FieldVector.get(): Index out of bounds') + ) { + runAsIfProver(transaction, index); + console.error( + 'This is likely due to a mismatch between the circuit at compile and proving time.\n' + ); + } + throw err; } finally { memoizationContext.leave(id); @@ -159,7 +171,6 @@ function getZkappProver({ methodName, ZkappClass }: LazyProof) { } // for debugging prove/compile discrepancies -// TODO run this automatically when detecting some problem function runAsIfProver(transaction: ZkappCommand, index: number) { let accountUpdate = transaction.accountUpdates[index]; @@ -209,6 +220,9 @@ function runAsIfProver(transaction: ZkappCommand, index: number) { withWitness: true, snarkContext: { proverData, inAnalyze: true }, expectedConstraints: metadata.expectedConstraints, + unexpectedConstraintMessage: + 'Constraint generated during prove() was different than the constraint generated at this location in compile().\n' + + 'See the stack trace below for where this constraint originated.', } ); } diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index 88d8a0ab36..1719d892d5 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -9,7 +9,7 @@ import { } from './gates.js'; import { MlArray, MlBool, MlOption, MlString, MlTuple } from './ml/base.js'; import { SnarkContext, snarkContext } from './provable-context.js'; -import { assertDeepEqual, deepEqual } from './util/nested.js'; +import { assertDeepEqual } from './util/nested.js'; export { runCircuit, SnarkyConstraint, ConstraintLog, MlConstraintSystem }; @@ -19,11 +19,13 @@ function runCircuit( withWitness, evalConstraints, expectedConstraints, + unexpectedConstraintMessage, snarkContext: ctx = {}, }: { withWitness: boolean; evalConstraints?: boolean; expectedConstraints?: ConstraintLog[]; + unexpectedConstraintMessage?: string; snarkContext?: SnarkContext; } ) { @@ -35,7 +37,7 @@ function runCircuit( numInputs, MlBool(evalConstraints ?? true), MlBool(withWitness), - MlOption((_label, maybeConstraint) => { + MlOption(function collectConstraints(_label, maybeConstraint) { let mlConstraint = MlOption.from(maybeConstraint); if (mlConstraint === undefined) return; let constraintLog = getGateTypeAndData(mlConstraint); @@ -43,7 +45,12 @@ function runCircuit( if (expectedConstraints !== undefined) { let expected = expectedConstraints[constraints.length - 1]; - assertDeepEqual(constraintLog, expected, 'constraint mismatch'); + assertDeepEqual( + constraintLog, + expected, + unexpectedConstraintMessage ?? + 'Generated constraint generated did not match expected constraint' + ); } }) ); diff --git a/src/lib/util/nested.ts b/src/lib/util/nested.ts index bee161d8fd..4ab3c55a5b 100644 --- a/src/lib/util/nested.ts +++ b/src/lib/util/nested.ts @@ -15,9 +15,8 @@ function assertDeepEqual(actual: Nested, expected: Nested, message?: string) { (BigInt.prototype as any).toJSON = function () { return this.toString(); }; - let fullMessage = `assertDeepEqual failed: ${message ?? ''} - -Inputs: + let fullMessage = `${message ? `${message}\n\n` : ''}Deep equality failed: + actual: ${JSON.stringify(actual)} expected: ${JSON.stringify(expected)} `; From ad20e1c644f7aa8ab007d27b56efea3f86aaff35 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 11:47:04 +0100 Subject: [PATCH 16/27] show traces to constraints in both versions --- src/lib/mina/zkapp-proof.ts | 45 ++++++++++++++++++++---------- src/lib/provable-context-debug.ts | 46 +++++++++++++++++++++++-------- src/lib/util/nested.ts | 35 +++++++++++++++++++---- 3 files changed, 93 insertions(+), 33 deletions(-) diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts index 8f573bcd24..347bb5823d 100644 --- a/src/lib/mina/zkapp-proof.ts +++ b/src/lib/mina/zkapp-proof.ts @@ -17,12 +17,14 @@ import { Empty, Proof, dummyBase64Proof, + emptyWitness, methodArgumentsToVars, + synthesizeMethodArguments, } from '../proof_system.js'; import { runCircuit } from '../provable-context-debug.js'; import { Provable, memoizationContext } from '../provable.js'; -export { addMissingProofs, runAsIfProver }; +export { addMissingProofs }; type AccountUpdateProved = AccountUpdate & { lazyAuthorization?: LazySignature; @@ -131,12 +133,8 @@ async function createZkappProof( err instanceof Error && err.message.includes('FieldVector.get(): Index out of bounds') ) { - runAsIfProver(transaction, index); - console.error( - 'This is likely due to a mismatch between the circuit at compile and proving time.\n' - ); + debugInconsistentConstraint(transaction, index); } - throw err; } finally { memoizationContext.leave(id); @@ -172,7 +170,7 @@ function getZkappProver({ methodName, ZkappClass }: LazyProof) { // for debugging prove/compile discrepancies -function runAsIfProver(transaction: ZkappCommand, index: number) { +function debugInconsistentConstraint(transaction: ZkappCommand, index: number) { let accountUpdate = transaction.accountUpdates[index]; accountUpdate = AccountUpdate.clone(accountUpdate); @@ -183,19 +181,33 @@ function runAsIfProver(transaction: ZkappCommand, index: number) { let { methodName, ZkappClass, args, memoized, blindingValue } = accountUpdate.lazyAuthorization; - let metadata = ZkappClass._methodMetadata?.[methodName]; let methodIntf = ZkappClass._methods?.find( (m) => m.methodName === methodName ); - assert( - metadata !== undefined && methodIntf !== undefined, - `No metadata found for zkapp method ${methodName}()` + // run circuit in compile mode to get expected constraints + let { constraints: expectedConstraints } = runCircuit( + () => { + let [pk, tid, ...otherArgs] = synthesizeMethodArguments( + methodIntf!, + true + ) as any[]; + let publicInput = emptyWitness(ZkappPublicInput); + + let instance = new ZkappClass(pk, tid); + (instance as any)[methodName](publicInput, ...otherArgs); + }, + { + withWitness: false, + snarkContext: { inAnalyze: true }, + createDebugTraces: true, + } ); let publicInput = accountUpdate.toPublicInput(); let proverData = { transaction, accountUpdate, index }; + // and a second time in prove mode to get actual constraints runCircuit( () => { let id = memoizationContext.enter({ @@ -208,10 +220,13 @@ function runAsIfProver(transaction: ZkappCommand, index: number) { [accountUpdate.publicKey, accountUpdate.tokenId, ...args], methodIntf! ).args; - publicInput = Provable.witness(ZkappPublicInput, () => publicInput); + let publicInput_ = Provable.witness( + ZkappPublicInput, + () => publicInput + ); let instance = new ZkappClass(pk, tid); - (instance as any)[methodName](publicInput, ...otherArgs); + (instance as any)[methodName](publicInput_, ...otherArgs); } finally { memoizationContext.leave(id); } @@ -219,10 +234,10 @@ function runAsIfProver(transaction: ZkappCommand, index: number) { { withWitness: true, snarkContext: { proverData, inAnalyze: true }, - expectedConstraints: metadata.expectedConstraints, + expectedConstraints, unexpectedConstraintMessage: 'Constraint generated during prove() was different than the constraint generated at this location in compile().\n' + - 'See the stack trace below for where this constraint originated.', + 'See the stack traces below for where this constraint originated.', } ); } diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index 1719d892d5..4a7c36f8bd 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -9,7 +9,7 @@ import { } from './gates.js'; import { MlArray, MlBool, MlOption, MlString, MlTuple } from './ml/base.js'; import { SnarkContext, snarkContext } from './provable-context.js'; -import { assertDeepEqual } from './util/nested.js'; +import { deepEqual, stringify } from './util/nested.js'; export { runCircuit, SnarkyConstraint, ConstraintLog, MlConstraintSystem }; @@ -17,16 +17,18 @@ function runCircuit( main: () => void, { withWitness, - evalConstraints, + evalConstraints = true, expectedConstraints, unexpectedConstraintMessage, snarkContext: ctx = {}, + createDebugTraces = false, }: { withWitness: boolean; evalConstraints?: boolean; expectedConstraints?: ConstraintLog[]; unexpectedConstraintMessage?: string; snarkContext?: SnarkContext; + createDebugTraces?: boolean; } ) { const snarkyState = Snarky.lowLevel.state; @@ -35,22 +37,37 @@ function runCircuit( let [, state, input, aux, system] = Snarky.lowLevel.createState( numInputs, - MlBool(evalConstraints ?? true), + MlBool(evalConstraints), MlBool(withWitness), MlOption(function collectConstraints(_label, maybeConstraint) { let mlConstraint = MlOption.from(maybeConstraint); if (mlConstraint === undefined) return; - let constraintLog = getGateTypeAndData(mlConstraint); - constraints.push(constraintLog); + let constraint = getGateTypeAndData(mlConstraint); + let debug = createDebugTraces ? new Error(constraint.type) : undefined; + constraints.push({ constraint, debug }); if (expectedConstraints !== undefined) { let expected = expectedConstraints[constraints.length - 1]; - assertDeepEqual( - constraintLog, - expected, + let ok = deepEqual(constraint, expected.constraint); + if (ok) return; + + let message = unexpectedConstraintMessage ?? - 'Generated constraint generated did not match expected constraint' - ); + 'Generated constraint generated did not match expected constraint..\n' + + 'See the stack traces below for where this constraint originated.'; + + let expectedStackTrace = + expected.debug?.stack !== undefined + ? `\nStack trace for the expected constraint:\n\n${expected.debug.stack}\n` + : ''; + + let fullMessage = `${message}\n\nDeep equality failed:\n +actual: ${stringify(constraint)} +expected: ${stringify(expected.constraint)} +${expectedStackTrace} +Stack trace for the actual constraint: +`; + throw Error(fullMessage); } }) ); @@ -79,7 +96,10 @@ class MlConstraintSystem { // opaque } -type ConstraintLog = { type: ConstraintType; data: any }; +type ConstraintLog = { + constraint: { type: ConstraintType; data: any }; + debug?: Error; +}; type ConstraintType = | KimchiGateTypeString @@ -88,7 +108,9 @@ type ConstraintType = | 'Square' | 'R1CS'; -function getGateTypeAndData(constraint: SnarkyConstraint): ConstraintLog { +function getGateTypeAndData( + constraint: SnarkyConstraint +): ConstraintLog['constraint'] { let [, basic] = constraint; switch (basic[1][1].c) { case SnarkyConstraintType.Boolean: diff --git a/src/lib/util/nested.ts b/src/lib/util/nested.ts index 4ab3c55a5b..85e03555cd 100644 --- a/src/lib/util/nested.ts +++ b/src/lib/util/nested.ts @@ -1,4 +1,4 @@ -export { assertDeepEqual, deepEqual }; +export { assertDeepEqual, deepEqual, stringify }; type Nested = | number @@ -12,13 +12,10 @@ type Nested = function assertDeepEqual(actual: Nested, expected: Nested, message?: string) { if (!deepEqual(actual, expected)) { - (BigInt.prototype as any).toJSON = function () { - return this.toString(); - }; let fullMessage = `${message ? `${message}\n\n` : ''}Deep equality failed: -actual: ${JSON.stringify(actual)} -expected: ${JSON.stringify(expected)} +actual: ${stringify(actual)} +expected: ${stringify(expected)} `; throw Error(fullMessage); } @@ -44,3 +41,29 @@ function deepEqual(a: Nested, b: Nested): boolean { } return true; } + +function stringify(x: Nested): string { + if (typeof x === 'object') { + if (x === null) return 'null'; + if (Array.isArray(x)) return `[${x.map(stringify).join(', ')}]`; + let result = '{ '; + for (let key in x) { + if (result.length > 2) result += ', '; + result += `${key}: ${stringify(x[key])}`; + } + return result + ' }'; + } + switch (typeof x) { + case 'string': + return `"${x}"`; + case 'bigint': + return `${x.toString()}n`; + case 'number': + case 'boolean': + return x.toString(); + case 'undefined': + return 'undefined'; + default: + throw Error(`Unexpected type ${typeof x}`); + } +} From b61599181ee8f122047b140ebe38eb2d71480505 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 12:29:49 +0100 Subject: [PATCH 17/27] seems more correct but I'm not sure --- src/lib/mina/zkapp-proof.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts index 347bb5823d..ef81506985 100644 --- a/src/lib/mina/zkapp-proof.ts +++ b/src/lib/mina/zkapp-proof.ts @@ -233,7 +233,7 @@ function debugInconsistentConstraint(transaction: ZkappCommand, index: number) { }, { withWitness: true, - snarkContext: { proverData, inAnalyze: true }, + snarkContext: { proverData, inProver: true }, expectedConstraints, unexpectedConstraintMessage: 'Constraint generated during prove() was different than the constraint generated at this location in compile().\n' + From 73b17e04f92b79b8ae7fa8583d9888dc788c5e47 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 12:53:56 +0100 Subject: [PATCH 18/27] helper to print account update layout --- src/lib/account_update.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index db5abf8d7b..4051659a1b 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -1096,11 +1096,37 @@ class AccountUpdate implements Types.AccountUpdate { return { accountUpdate, calls }; } + toPrettyLayout() { + let indent = 0; + let layout = ''; + let i = 0; + + let print = (a: AccountUpdate) => { + layout += + ' '.repeat(indent) + + `AccountUpdate(${i}, ${a.label || ''}, ${ + a.children.callsType.type + })` + + '\n'; + i++; + indent += 2; + for (let child of a.children.accountUpdates) { + print(child); + } + indent -= 2; + }; + + print(this); + return layout; + } + static defaultAccountUpdate(address: PublicKey, tokenId?: Field) { return new AccountUpdate(Body.keepAll(address, tokenId)); } static dummy() { - return new AccountUpdate(Body.dummy()); + let dummy = new AccountUpdate(Body.dummy()); + dummy.label = 'Dummy'; + return dummy; } isDummy() { return this.body.publicKey.isEmpty(); From e3cfaf91bdb1c9fbb10522439f6b8df69c351cb7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 12:55:46 +0100 Subject: [PATCH 19/27] fix the damn bug --- src/lib/account_update.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/account_update.ts b/src/lib/account_update.ts index 4051659a1b..3e95bd4177 100644 --- a/src/lib/account_update.ts +++ b/src/lib/account_update.ts @@ -1347,6 +1347,7 @@ class AccountUpdate implements Types.AccountUpdate { accountUpdate.body.mayUseToken.inheritFromParent.assertFalse(); return; } + accountUpdate.children.callsType = { type: 'None' }; let childArray: AccountUpdatesLayout[] = typeof childLayout === 'number' ? Array(childLayout).fill(AccountUpdate.Layout.NoChildren) From 061cad091147a470248d090da83dac7dcdecbb5d Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 13:00:28 +0100 Subject: [PATCH 20/27] [dex] remove obsolete scare comments --- src/examples/zkapps/dex/dex.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/examples/zkapps/dex/dex.ts b/src/examples/zkapps/dex/dex.ts index 0f81f0a0c3..44ac6b5178 100644 --- a/src/examples/zkapps/dex/dex.ts +++ b/src/examples/zkapps/dex/dex.ts @@ -440,10 +440,6 @@ class TokenContract extends SmartContract { to: PublicKey, amount: UInt64 ) { - // TODO: THIS IS INSECURE. The proper version has a prover error (compile != prove) that must be fixed - // this.approve(zkappUpdate, AccountUpdate.Layout.AnyChildren); - - // THIS IS HOW IT SHOULD BE DONE: // approve a layout of two grandchildren, both of which can't inherit the token permission let { StaticChildren, AnyChildren } = AccountUpdate.Layout; this.approve(zkappUpdate, StaticChildren(AnyChildren, AnyChildren)); From 93568b811f4cf1ee290a20ae6f0aa8ff6d747818 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 13:00:37 +0100 Subject: [PATCH 21/27] minor example tweak --- src/examples/simple_zkapp.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/examples/simple_zkapp.ts b/src/examples/simple_zkapp.ts index 26f379770d..bd17d6eae4 100644 --- a/src/examples/simple_zkapp.ts +++ b/src/examples/simple_zkapp.ts @@ -91,6 +91,8 @@ if (doProofs) { console.time('compile'); await SimpleZkapp.compile(); console.timeEnd('compile'); +} else { + SimpleZkapp.analyzeMethods(); } console.log('deploy'); From 9127ae6296ad7978a3e262a02735ca300d69153f Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 13:03:05 +0100 Subject: [PATCH 22/27] submodules --- src/bindings | 2 +- src/mina | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bindings b/src/bindings index a884dc593d..9f851339d3 160000 --- a/src/bindings +++ b/src/bindings @@ -1 +1 @@ -Subproject commit a884dc593dbab69e55ab9602b998ec12dfc3a288 +Subproject commit 9f851339d3895f04c1704009751b4e068cc08f62 diff --git a/src/mina b/src/mina index 2a968c8347..f867de6429 160000 --- a/src/mina +++ b/src/mina @@ -1 +1 @@ -Subproject commit 2a968c83477ed9f9e3b30a02cc357e541b76dcac +Subproject commit f867de6429dd433b483b7e5b8c668166a7e275c8 From f19076ac44137b452573710838592bd39a9ac2ac Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 11 Jan 2024 13:21:40 +0100 Subject: [PATCH 23/27] changelog --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd76e3380e..cd308145b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased](https://github.com/o1-labs/o1js/compare/08ba27329...HEAD) +### Fixed + +- Fix approving of complex account update layouts https://github.com/o1-labs/o1js/pull/1364 + ## [0.15.2](https://github.com/o1-labs/o1js/compare/1ad7333e9e...08ba27329) ### Fixed -- Fix bug in `Hash.hash()` which always resulted in an error. https://github.com/o1-labs/o1js/pull/1346 +- Fix bug in `Hash.hash()` which always resulted in an error https://github.com/o1-labs/o1js/pull/1346 ## [0.15.1](https://github.com/o1-labs/o1js/compare/1ad7333e9e...19115a159) From e9979d9a73350471a2c9677ccdc49d84c6f5531a Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 17 Jan 2024 10:12:59 +0100 Subject: [PATCH 24/27] reduce diff to main where changes weren't necessary --- src/lib/proof_system.ts | 27 ++++++++------------------- src/lib/zkapp.ts | 5 +---- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/src/lib/proof_system.ts b/src/lib/proof_system.ts index ce65ae23bb..e123818c4c 100644 --- a/src/lib/proof_system.ts +++ b/src/lib/proof_system.ts @@ -11,7 +11,6 @@ import { MlFeatureFlags, Gate, GateType, - Snarky, } from '../snarky.js'; import { Field, Bool } from './core.js'; import { @@ -26,7 +25,7 @@ import { } from './circuit_value.js'; import { Provable } from './provable.js'; import { assert, prettifyStacktracePromise } from './errors.js'; -import { constraintSystemFromJson, snarkContext } from './provable-context.js'; +import { snarkContext } from './provable-context.js'; import { hashConstant } from './hash.js'; import { MlArray, MlBool, MlResult, MlPair } from './ml/base.js'; import { MlFieldArray, MlFieldConstArray } from './ml/fields.js'; @@ -38,7 +37,6 @@ import { parseHeader, } from './proof-system/prover-keys.js'; import { setSrsCache, unsetSrsCache } from '../bindings/crypto/bindings/srs.js'; -import { runCircuit } from './provable-context-debug.js'; // public API export { @@ -734,22 +732,13 @@ function analyzeMethod( methodIntf: MethodInterface, method: (...args: any) => T ) { - let { constraints, system } = runCircuit( - () => { - let args = synthesizeMethodArguments(methodIntf, true); - let publicInput = emptyWitness(publicInputType); - if (publicInputType === Undefined || publicInputType === Void) { - method(...args); - } else { - method(publicInput, ...args); - } - }, - { withWitness: false, snarkContext: { inAnalyze: true } } - ); - let digest = Snarky.lowLevel.constraintSystem.digest(system); - let csJson = Snarky.lowLevel.constraintSystem.toJson(system); - let cs = constraintSystemFromJson(csJson); - return { constraints, rows: cs.gates.length, digest, ...cs }; + return Provable.constraintSystem(() => { + let args = synthesizeMethodArguments(methodIntf, true); + let publicInput = emptyWitness(publicInputType); + if (publicInputType === Undefined || publicInputType === Void) + return method(...args); + return method(publicInput, ...args); + }); } function picklesRuleFromFunction( diff --git a/src/lib/zkapp.ts b/src/lib/zkapp.ts index fe88881f1d..d37cc7d96c 100644 --- a/src/lib/zkapp.ts +++ b/src/lib/zkapp.ts @@ -56,7 +56,6 @@ import { snarkContext, } from './provable-context.js'; import { Cache } from './proof-system/cache.js'; -import type { ConstraintLog } from './provable-context-debug.js'; // external API export { @@ -617,7 +616,6 @@ class SmartContract { digest: string; hasReturn: boolean; gates: Gate[]; - expectedConstraints?: ConstraintLog[]; } >; // keyed by method name static _provers?: Pickles.Prover[]; @@ -1189,7 +1187,7 @@ super.init(); for (let methodIntf of methodIntfs) { let accountUpdate: AccountUpdate; let hasReturn = false; - let { rows, digest, gates, constraints } = analyzeMethod( + let { rows, digest, gates } = analyzeMethod( ZkappPublicInput, methodIntf, (publicInput, publicKey, tokenId, ...args) => { @@ -1208,7 +1206,6 @@ super.init(); digest, hasReturn, gates, - expectedConstraints: constraints, }; } } finally { From 68487d7c6f8d4ca8829abedfeb7e7bab0bd3ba1b Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 17 Jan 2024 10:35:44 +0100 Subject: [PATCH 25/27] add docs and changelog --- CHANGELOG.md | 5 +++++ src/lib/provable-context-debug.ts | 19 +++++++++++++++++++ src/lib/provable-context.ts | 7 +++++++ 3 files changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dadf1323e4..f61df445a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **SHA256 hash function** exposed via `Hash.SHA2_256` or `Gadgets.SHA256`. https://github.com/o1-labs/o1js/pull/1285 +### Changed + +- Massively improved error reporting when detecting mismatches in the constraints generated in `compile()` vs `prove()` https://github.com/o1-labs/o1js/pull/1363 + - Made possible by new internal tooling to collect constraints, along with their stack trace, when they are added by snarky . + ### Fixed - Fix approving of complex account update layouts https://github.com/o1-labs/o1js/pull/1364 diff --git a/src/lib/provable-context-debug.ts b/src/lib/provable-context-debug.ts index 4a7c36f8bd..b560cf9baa 100644 --- a/src/lib/provable-context-debug.ts +++ b/src/lib/provable-context-debug.ts @@ -1,3 +1,11 @@ +/** + * This contains a runner for provable code which is has advanced powers + * compared to the provable code runners found in `./provable-context.ts`, + * and can be used for debugging expected vs received constraints. + * + * To achieve this, it uses low-level hooks into snarky's internal state and + * contains code to interpret constraints in the format that snarky logs them in. + */ import { Snarky } from '../snarky.js'; import { prettifyStacktrace } from './errors.js'; import { FieldConst, FieldVar } from './field.js'; @@ -13,6 +21,17 @@ import { deepEqual, stringify } from './util/nested.js'; export { runCircuit, SnarkyConstraint, ConstraintLog, MlConstraintSystem }; +/** + * Run a circuit `main()` with various advanced options: + * + * - `withWitness`: whether to run witness blocks and collect the witness + * - `evalConstraints`: whether to throw on unsatisfied constraints + * - `expectedConstraints`: a list of constraints from a precious `runCircuit()` call to compare against + * - `unexpectedConstraintMessage`: a message to throw if the constraints don't match + * - `snarkContext`: extra inputs to the {@link SnarkContext} this runs in + * - `createDebugTraces`: whether to create a stack trace for each constraint, that can be be printed + * in a future run to show where an expected constraint originated + */ function runCircuit( main: () => void, { diff --git a/src/lib/provable-context.ts b/src/lib/provable-context.ts index 224e67cec8..c8fdf2f236 100644 --- a/src/lib/provable-context.ts +++ b/src/lib/provable-context.ts @@ -1,3 +1,10 @@ +/** + * This contains functions to manage the global context for provable code: + * + * - A global `snarkContext` which tells us whether we're in a provable context, + * and contains a small amount of global state on top of snarky's internal state. + * - Various "runners" which execute provable code in different modes. + */ import { Context } from './global-context.js'; import { Gate, From dff07c4689a7f06f0c965795fd2699de8529c623 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 17 Jan 2024 11:11:02 +0100 Subject: [PATCH 26/27] more cases in which to check inconsistent constraints --- src/lib/mina/zkapp-proof.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/mina/zkapp-proof.ts b/src/lib/mina/zkapp-proof.ts index ef81506985..3c9f57d19a 100644 --- a/src/lib/mina/zkapp-proof.ts +++ b/src/lib/mina/zkapp-proof.ts @@ -131,7 +131,9 @@ async function createZkappProof( ); if ( err instanceof Error && - err.message.includes('FieldVector.get(): Index out of bounds') + (err.message.includes('FieldVector.get(): Index out of bounds') || + err.message.includes('rest of division by vanishing polynomial') || + err.message.includes('Constraint unsatisfied')) ) { debugInconsistentConstraint(transaction, index); } From e49b7881b14b4584348cacb04a8a2c54f870846b Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 17 Jan 2024 11:11:35 +0100 Subject: [PATCH 27/27] bindings --- src/bindings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bindings b/src/bindings index 9f851339d3..9830a95564 160000 --- a/src/bindings +++ b/src/bindings @@ -1 +1 @@ -Subproject commit 9f851339d3895f04c1704009751b4e068cc08f62 +Subproject commit 9830a955642adf97a09719570c13272d264b2eac