From fc80ecd4f403ad0397c4d94428998de7323c3e23 Mon Sep 17 00:00:00 2001 From: Soleimani193 Date: Fri, 13 Sep 2024 16:28:03 +0200 Subject: [PATCH] debugging --- .../column/verifiercol/expand_verifcol.go | 90 ++++++++++++++ .../column/verifiercol/from_accessors.go | 2 +- .../compiler/splitter/stitcher/constraints.go | 18 ++- .../compiler/splitter/stitcher/stitcher.go | 113 +++++++++++------- .../splitter/stitcher/stitcher_test.go | 61 ++++++++++ 5 files changed, 238 insertions(+), 46 deletions(-) create mode 100644 prover/protocol/column/verifiercol/expand_verifcol.go diff --git a/prover/protocol/column/verifiercol/expand_verifcol.go b/prover/protocol/column/verifiercol/expand_verifcol.go new file mode 100644 index 000000000..620e47691 --- /dev/null +++ b/prover/protocol/column/verifiercol/expand_verifcol.go @@ -0,0 +1,90 @@ +package verifiercol + +import ( + "github.com/consensys/gnark/frontend" + "github.com/consensys/zkevm-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/zkevm-monorepo/prover/maths/common/vector" + "github.com/consensys/zkevm-monorepo/prover/maths/field" + "github.com/consensys/zkevm-monorepo/prover/protocol/ifaces" + "github.com/consensys/zkevm-monorepo/prover/protocol/wizard" +) + +// compile check to enforce the struct to belong to the corresponding interface +var _ VerifierCol = ExpandedVerifCol{} + +type ExpandedVerifCol struct { + Verifiercol VerifierCol + Expansion int +} + +// Round returns the round ID of the column and implements the [ifaces.Column] +// interface. +func (ex ExpandedVerifCol) Round() int { + return ex.Verifiercol.Round() +} + +// GetColID returns the column ID +func (ex ExpandedVerifCol) GetColID() ifaces.ColID { + return ifaces.ColIDf("Expanded_%v", ex.Verifiercol.GetColID()) +} + +// MustExists implements the [ifaces.Column] interface and always returns true. +func (ex ExpandedVerifCol) MustExists() { + ex.Verifiercol.MustExists() +} + +// Size returns the size of the colum and implements the [ifaces.Column] +// interface. +func (ex ExpandedVerifCol) Size() int { + return ex.Verifiercol.Size() * ex.Expansion +} + +// GetColAssignment returns the assignment of the current column +func (ex ExpandedVerifCol) GetColAssignment(run ifaces.Runtime) ifaces.ColAssignment { + assi := ex.Verifiercol.GetColAssignment(run) + values := make([][]field.Element, ex.Expansion) + for j := range values { + values[j] = smartvectors.IntoRegVec(assi) + } + res := vector.Interleave(values...) + return smartvectors.NewRegular(res) +} + +// GetColAssignment returns a gnark assignment of the current column +func (ex ExpandedVerifCol) GetColAssignmentGnark(run ifaces.GnarkRuntime) []frontend.Variable { + + assi := ex.Verifiercol.GetColAssignmentGnark(run) + res := make([]frontend.Variable, ex.Size()) + for i := 0; i < len(assi); i++ { + for j := 0; j < ex.Expansion; j++ { + res[j+i*ex.Expansion] = assi[i] + } + } + return res +} + +// GetColAssignmentAt returns a particular position of the column +func (ex ExpandedVerifCol) GetColAssignmentAt(run ifaces.Runtime, pos int) field.Element { + return ex.Verifiercol.GetColAssignmentAt(run, pos/ex.Expansion) +} + +// GetColAssignmentGnarkAt returns a particular position of the column in a gnark circuit +func (ex ExpandedVerifCol) GetColAssignmentGnarkAt(run ifaces.GnarkRuntime, pos int) frontend.Variable { + + return ex.GetColAssignmentGnarkAt(run, pos/ex.Expansion) +} + +// IsComposite implements the [ifaces.Column] interface +func (ex ExpandedVerifCol) IsComposite() bool { + return ex.Verifiercol.IsComposite() +} + +// String implements the [symbolic.Metadata] interface +func (ex ExpandedVerifCol) String() string { + return ex.Verifiercol.String() +} + +// Split implements the [VerifierCol] interface +func (ex ExpandedVerifCol) Split(_ *wizard.CompiledIOP, from, to int) ifaces.Column { + return ex.Verifiercol +} diff --git a/prover/protocol/column/verifiercol/from_accessors.go b/prover/protocol/column/verifiercol/from_accessors.go index 2eea330c7..2f2d8172a 100644 --- a/prover/protocol/column/verifiercol/from_accessors.go +++ b/prover/protocol/column/verifiercol/from_accessors.go @@ -123,7 +123,7 @@ func (f FromAccessors) IsComposite() bool { return false } -// IsComposite implements the [symbolic.Metadata] interface +// String implements the [symbolic.Metadata] interface func (f FromAccessors) String() string { return string(f.GetColID()) } diff --git a/prover/protocol/compiler/splitter/stitcher/constraints.go b/prover/protocol/compiler/splitter/stitcher/constraints.go index 1ce3199d1..c12043395 100644 --- a/prover/protocol/compiler/splitter/stitcher/constraints.go +++ b/prover/protocol/compiler/splitter/stitcher/constraints.go @@ -3,6 +3,7 @@ package stitcher import ( "github.com/consensys/zkevm-monorepo/prover/protocol/coin" "github.com/consensys/zkevm-monorepo/prover/protocol/column" + "github.com/consensys/zkevm-monorepo/prover/protocol/column/verifiercol" "github.com/consensys/zkevm-monorepo/prover/protocol/ifaces" "github.com/consensys/zkevm-monorepo/prover/protocol/query" "github.com/consensys/zkevm-monorepo/prover/protocol/variables" @@ -59,7 +60,8 @@ func (ctx stitchingContext) LocalGlobalConstraints() { switch q := q.(type) { case query.LocalConstraint: board = q.Board() - // detect if the expression is over the eligible columns. + // detect if the expression is eligible; + // i.e., it contains columns of proper size with status Precomputed, committed, or verifiercol. if !isExprEligible(ctx, board) { continue } @@ -96,6 +98,15 @@ func (ctx stitchingContext) LocalGlobalConstraints() { // more detailed, such stitching column agrees with the the sub column up to a subsampling with offset zero. func getStitchingCol(ctx stitchingContext, col ifaces.Column) ifaces.Column { + switch m := col.(type) { + case verifiercol.VerifierCol: + scaling := ctx.MaxSize / col.Size() + return verifiercol.ExpandedVerifCol{ + Verifiercol: m, + Expansion: scaling, + } + } + // Extract the assumedly single col natural := column.RootParents(col)[0] @@ -116,7 +127,10 @@ func queryName(oldQ ifaces.QueryID) ifaces.QueryID { return ifaces.QueryIDf("%v_STITCHER", oldQ) } -// it adjust the expression, that is among sub columns, by replacing the sub columns with their stitching columns. +// it adjusts the expression, that is among sub columns, by replacing the sub columns with their stitching columns. +// for the verfiercol instead of stitching, they are expanded to reach the proper size. +// This is due to the fact that the verifiercols are not tracked by the compiler and can not be stitched +// via [scanAndClassifyEligibleColumns]. func (ctx *stitchingContext) adjustExpression( expr *symbolic.Expression, isGlobalConstraint bool, diff --git a/prover/protocol/compiler/splitter/stitcher/stitcher.go b/prover/protocol/compiler/splitter/stitcher/stitcher.go index 0b4204382..11178f040 100644 --- a/prover/protocol/compiler/splitter/stitcher/stitcher.go +++ b/prover/protocol/compiler/splitter/stitcher/stitcher.go @@ -45,9 +45,9 @@ type stitching struct { // the round in which the sub columns are committed. round int // status of the sub columns - isPreComputed bool - //TBD: supporting verifierDefined via expansion - // for the moment it is handled similar to committed columns (which is not very efficient). + // the only valid status for the eligible sub columns are; + // committed, Precomputed, VerifierDefined + status column.Status } // Stitcher applies the stitching over the eligible sub columns and adjusts the constraints accordingly. @@ -57,13 +57,10 @@ func Stitcher(minSize, maxSize int) func(comp *wizard.CompiledIOP) { // it creates stitchings from the eligible columns and commits to the them. ctx := newStitcher(comp, minSize, maxSize) - // ignore the constraints over the subColumns. - // ctx.IgnoreConstraintsOverSubColumns() - // adjust the constraints accordingly over the stitchings of the sub columns. ctx.constraints() - // it assign the stitching columns and delete the assignment of the sub columns. + // it assigns the stitching columns and delete the assignment of the sub columns. comp.SubProvers.AppendToInner(comp.NumRounds()-1, func(run *wizard.ProverRuntime) { for round := range comp.NumRounds() { for subCol, _ := range ctx.Stitchings[round].BySubCol { @@ -109,16 +106,26 @@ func (ctx *stitchingContext) ScanStitchCommit() { var ( precomputedCols = make([]ifaces.Column, 0, len(cols)) - normalCols = make([]ifaces.Column, 0, len(cols)) + committedCols = make([]ifaces.Column, 0, len(cols)) ) - // collect the precomputedColumns and normal columns. + // collect the the columns with valid status; Precomputed, committed + // verifierDefined is valid but is not present in the compiler trace we handle it directly during the constraints. for _, col := range cols { - if ctx.comp.Columns.Status(col.GetColID()) == column.Precomputed { + status := ctx.comp.Columns.Status(col.GetColID()) + switch status { + case column.Precomputed: precomputedCols = append(precomputedCols, col) - } else { - normalCols = append(normalCols, col) + case column.Committed: + committedCols = append(committedCols, col) + + default: + utils.Panic("found the column %v with the invalid status %v for stitching", col.GetColID(), status.String()) } + + // Mark it as ignored, so that it is no longer considered as + // queryable (since we are replacing it with its stitching). + ctx.comp.Columns.MarkAsIgnored(col.GetColID()) } if len(precomputedCols) != 0 { @@ -128,23 +135,23 @@ func (ctx *stitchingContext) ScanStitchCommit() { for _, group := range preComputedGroups { // prepare a group for stitching stitching := stitching{ - subCol: group, - round: round, - isPreComputed: true, + subCol: group, + round: round, + status: column.Precomputed, } // stitch the group ctx.stitchGroup(stitching) } } - if len(normalCols) != 0 { - normalGroups := groupCols(normalCols, ctx.MaxSize/size) + if len(committedCols) != 0 { + committedGroups := groupCols(committedCols, ctx.MaxSize/size) - for _, group := range normalGroups { + for _, group := range committedGroups { stitching := stitching{ - subCol: group, - round: round, - isPreComputed: false, + subCol: group, + round: round, + status: column.Committed, } ctx.stitchGroup(stitching) @@ -157,7 +164,7 @@ func (ctx *stitchingContext) ScanStitchCommit() { stopTimer := profiling.LogTimer("stitching compiler") defer stopTimer() for id, subColumns := range ctx.Stitchings[round].ByStitching { - // Trick, in order to compute the assignment of newName we + // Trick, in order to compute the assignment of stitching column, we // extract the witness of the interleaving of the grouped // columns. witnesses := make([]smartvectors.SmartVector, len(subColumns)) @@ -198,10 +205,6 @@ func scanAndClassifyEligibleColumns(ctx stitchingContext, round int) map[int][]i continue } - // Mark it as ignored, so that it is no longer considered as - // queryable (since we are replacing it with its stitching). - ctx.comp.Columns.MarkAsIgnored(colName) - // Initialization clause of `sizes` if _, ok := columnsBySize[col.Size()]; !ok { columnsBySize[col.Size()] = []ifaces.Column{} @@ -213,9 +216,9 @@ func scanAndClassifyEligibleColumns(ctx stitchingContext, round int) map[int][]i } // group the cols with the same size -func groupCols(cols []ifaces.Column, numToStick int) (groups [][]ifaces.Column) { +func groupCols(cols []ifaces.Column, numToStitch int) (groups [][]ifaces.Column) { - numGroups := utils.DivCeil(len(cols), numToStick) + numGroups := utils.DivCeil(len(cols), numToStitch) groups = make([][]ifaces.Column, numGroups) size := cols[0].Size() @@ -227,13 +230,13 @@ func groupCols(cols []ifaces.Column, numToStick int) (groups [][]ifaces.Column) col.GetColID(), col.Size(), cols[0].GetColID(), cols[0].Size(), ) } - groups[i/numToStick] = append(groups[i/numToStick], col) + groups[i/numToStitch] = append(groups[i/numToStitch], col) } lastGroup := &groups[len(groups)-1] zeroCol := verifiercol.NewConstantCol(field.Zero(), size) - for i := len(*lastGroup); i < numToStick; i++ { + for i := len(*lastGroup); i < numToStitch; i++ { *lastGroup = append(*lastGroup, zeroCol) } @@ -245,7 +248,7 @@ func groupedName(group []ifaces.Column) ifaces.ColID { for i := range fmtted { fmtted[i] = group[i].String() } - return ifaces.ColIDf("STICKER_%v", strings.Join(fmtted, "_")) + return ifaces.ColIDf("STITCHER_%v", strings.Join(fmtted, "_")) } // for a group of sub columns it creates their stitching. @@ -253,9 +256,11 @@ func (ctx *stitchingContext) stitchGroup(s stitching) { var ( group = s.subCol stitchingCol ifaces.Column + status = s.status ) // Declare the new columns - if s.isPreComputed { + switch status { + case column.Precomputed: values := make([][]field.Element, len(group)) for j := range values { values[j] = smartvectors.IntoRegVec(ctx.comp.Precomputed.MustGet(group[j].GetColID())) @@ -265,13 +270,16 @@ func (ctx *stitchingContext) stitchGroup(s stitching) { groupedName(group), smartvectors.NewRegular(assignement), ) - } else { + case column.Committed: stitchingCol = ctx.comp.InsertCommit( s.round, groupedName(s.subCol), ctx.MaxSize, ) + default: + panic("The status is not valid for the stitching") + } s.stitchingRes = stitchingCol @@ -315,22 +323,34 @@ func isColEligible(ctx stitchingContext, col ifaces.Column) bool { } // It checks if the expression is over a set of the columns eligible to the stitching. -// It panics if the expression includes a mixture of eligible columns and Proof/VerifiyingKey/Ignored status. +// Namely, it contains columns of proper size with status Precomputed, Committed, or Verifiercol. +// It panics if the expression includes a mixture of eligible columns and columns with status Proof/VerifiyingKey/Ignored. +// +// If all the columns are verifierCol the expression is not eligible to the compilation. +// This is an expected behavior, since the verifier checks such expression by itself. func isExprEligible(ctx stitchingContext, board symbolic.ExpressionBoard) bool { metadata := board.ListVariableMetadata() hasAtLeastOneEligible := false allAreEligible := true + allAreVeriferCol := true for i := range metadata { switch m := metadata[i].(type) { - case ifaces.Column: - b := isColEligible(ctx, m) - - hasAtLeastOneEligible = hasAtLeastOneEligible || b - allAreEligible = allAreEligible && b - - if m.Size() == 0 { - panic("found no columns in the expression") + // reminder: [verifiercol.VerifierCol] , [column.Natural] and [column.Shifted] + // all implement [ifaces.Column] + case ifaces.Column: // it is a Committed, Precomputed or verifierCol + natural := column.RootParents(m)[0] + switch natural.(type) { + case column.Natural: // then it is not a verifiercol + allAreVeriferCol = false + b := isColEligible(ctx, m) + + hasAtLeastOneEligible = hasAtLeastOneEligible || b + allAreEligible = allAreEligible && b + if m.Size() == 0 { + panic("found no columns in the expression") + } } + } } @@ -339,7 +359,14 @@ func isExprEligible(ctx stitchingContext, board symbolic.ExpressionBoard) bool { // 1. we expect no expression including Proof columns // 2. we expect no expression over ignored columns // 3. we expect no VerifiyingKey withing the stitching range. - panic("the expression is not valid, it incudes a mix of eligible columns and invalid columns i.e., Ignored,Proof, VerifingKey") + panic("the expression is not valid, it is mixed with invalid columns of status Proof/Ingnored/verifierKey") + } + if allAreVeriferCol { + // 4. we expect no expression involving only and only the verifierCols. + // We expect that this case wont happen. + // Otherwise should be handled in the [github.com/consensys/zkevm-monorepo/prover/protocol/query] package. + // Namely, Local/Global queries should be checked directly by the verifer. + panic("all the columns in the expression are verifierCols, unsupported by the compiler") } return hasAtLeastOneEligible diff --git a/prover/protocol/compiler/splitter/stitcher/stitcher_test.go b/prover/protocol/compiler/splitter/stitcher/stitcher_test.go index 7081edd83..463832ee0 100644 --- a/prover/protocol/compiler/splitter/stitcher/stitcher_test.go +++ b/prover/protocol/compiler/splitter/stitcher/stitcher_test.go @@ -5,8 +5,10 @@ import ( "github.com/consensys/zkevm-monorepo/prover/maths/common/smartvectors" "github.com/consensys/zkevm-monorepo/prover/maths/field" + "github.com/consensys/zkevm-monorepo/prover/protocol/accessors" "github.com/consensys/zkevm-monorepo/prover/protocol/coin" "github.com/consensys/zkevm-monorepo/prover/protocol/column" + "github.com/consensys/zkevm-monorepo/prover/protocol/column/verifiercol" "github.com/consensys/zkevm-monorepo/prover/protocol/compiler/dummy" "github.com/consensys/zkevm-monorepo/prover/protocol/compiler/splitter/stitcher" "github.com/consensys/zkevm-monorepo/prover/protocol/ifaces" @@ -49,6 +51,8 @@ func TestLocalEval(t *testing.T) { } comp := wizard.Compile(define, stitcher.Stitcher(4, 8)) + + //after stitcing-compilation we expect that the eligible columns and their relevant queries be ignored assert.Equal(t, column.Committed.String(), comp.Columns.Status("A").String()) assert.Equal(t, column.Ignored.String(), comp.Columns.Status("B").String()) assert.Equal(t, column.Committed.String(), comp.Columns.Status("C").String()) @@ -105,6 +109,7 @@ func TestGlobalConstraintFibonacci(t *testing.T) { define := func(builder *wizard.Builder) { // declare columns of different sizes a = builder.RegisterCommit("B", 4) + // a = verifiercol.NewConstantCol(field.One(), 4) b = builder.RegisterCommit("C", 8) c = builder.RegisterCommit("D", 16) @@ -122,6 +127,7 @@ func TestGlobalConstraintFibonacci(t *testing.T) { comp := wizard.Compile(define, stitcher.Stitcher(4, 8)) + //after stitcing-compilation we expect that the eligible columns and their relevant queries be ignored assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q1.ID), "q1 should be ignored") assert.Equal(t, false, comp.QueriesNoParams.IsIgnored(q2.ID), "q2 should not be ignored") assert.Equal(t, false, comp.QueriesNoParams.IsIgnored(q3.ID), "q3 should not be ignored") @@ -166,6 +172,7 @@ func TestLocalConstraintFibonacci(t *testing.T) { comp := wizard.Compile(define, stitcher.Stitcher(4, 8)) + //after stitcing-compilation we expect that the eligible columns and their relevant queries be ignored assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q1.ID), "q1 should be ignored") assert.Equal(t, false, comp.QueriesNoParams.IsIgnored(q2.ID), "q2 should not be ignored") assert.Equal(t, false, comp.QueriesNoParams.IsIgnored(q3.ID), "q3 should not be ignored") @@ -208,6 +215,7 @@ func TestGlobalMixedRounds(t *testing.T) { comp := wizard.Compile(define, stitcher.Stitcher(4, 8)) + //after stitcing-compilation we expect that the eligible columns and their relevant queries be ignored assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q0.ID), "q0 should be ignored") assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q1.ID), "q1 should be ignored") assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q2.ID), "q2 should be ignored") @@ -229,3 +237,56 @@ func TestGlobalMixedRounds(t *testing.T) { err := wizard.Verify(comp, proof) require.NoError(t, err) } + +func TestGlobalWithVerifCol(t *testing.T) { + var a, b, c, verifcol1, verifcol2 ifaces.Column + var q1, q2 query.GlobalConstraint + + define := func(builder *wizard.Builder) { + // declare columns of different sizes + a = builder.RegisterCommit("B", 4) + b = builder.RegisterCommit("C", 4) + // a new round + _ = builder.RegisterRandomCoin("COIN", coin.Field) + c = builder.RegisterCommit("D", 4) + // verifiercols + verifcol1 = verifiercol.NewConstantCol(field.NewElement(3), 4) + accessors := genAccessors([]int{1, 7, 5, 3}) + verifcol2 = verifiercol.NewFromAccessors(accessors, field.Zero(), 4) + + expr := symbolic.Sub(symbolic.Mul(a, verifcol1), b) + q1 = builder.GlobalConstraint("Q0", expr) + + expr = symbolic.Sub(symbolic.Add(a, verifcol2), c) + q2 = builder.GlobalConstraint("Q1", expr) + } + + comp := wizard.Compile(define, stitcher.Stitcher(4, 8)) + + //after stitcing-compilation we expect that the eligible columns and their relevant queries be ignored + assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q1.ID), "q1 should be ignored") + assert.Equal(t, true, comp.QueriesNoParams.IsIgnored(q2.ID), "q2 should be ignored") + + // manually compiles the comp + dummy.Compile(comp) + + proof := wizard.Prove(comp, func(assi *wizard.ProverRuntime) { + // Assigns all the columns + assi.AssignColumn(a.GetColID(), smartvectors.ForTest(1, 1, 2, 3)) + assi.AssignColumn(b.GetColID(), smartvectors.ForTest(3, 3, 6, 9)) + _ = assi.GetRandomCoinField("COIN") // triggers going to the next round + assi.AssignColumn(c.GetColID(), smartvectors.ForTest(2, 8, 7, 6)) + }) + + err := wizard.Verify(comp, proof) + require.NoError(t, err) + +} + +func genAccessors(a []int) (res []ifaces.Accessor) { + for i := range a { + t := accessors.NewConstant(field.NewElement(uint64(a[i]))) + res = append(res, t) + } + return res +}