getting the multiplicity table from compild iop

prover/protocol/compiler/lookup/compiler.go

@@ -131,9 +131,9 @@ func CompileLogDerivative(comp *wizard.CompiledIOP) {
 // The function also implictly reduces the conditionals over the Including table
 // be appending a "one" column on the included side and the filter on the
 // including side.
-func captureLookupTables(comp *wizard.CompiledIOP) MainLookupCtx {
+func captureLookupTables(comp *wizard.CompiledIOP) mainLookupCtx {
 
-	ctx := MainLookupCtx{
+	ctx := mainLookupCtx{
 		LookupTables:    [][]table{},
 		CheckedTables:   map[string][]table{},
 		IncludedFilters: map[string][]ifaces.Column{},

prover/protocol/compiler/lookup/context.go

@@ -6,9 +6,9 @@ import (
 	"github.com/consensys/linea-monorepo/prover/symbolic"
 )
 
-// MainLookupCtx stores the compilation context of all the lookup queries
+// mainLookupCtx stores the compilation context of all the lookup queries
 // altogether.
-type MainLookupCtx struct {
+type mainLookupCtx struct {
 
 	// LookupTables stores all the lookup table the compiler encounters. They are
 	// sorted in canonical order. This used to derive a determistic ordering
@@ -36,9 +36,6 @@ type MainLookupCtx struct {
 	// round is obtained by taking the max of the declaration Rounds of the
 	// Inclusion queries using the corresponding lookup table.
 	Rounds map[string]int
-
-	// the name of the tables
-	AllTableNames []string
 }
 
 // SingleTableCtx stores the compilation context for a single lookup query

prover/protocol/distributed/compiler/inclusion/context.go

@@ -0,0 +1,41 @@
+package inclusion
+
+import "github.com/consensys/linea-monorepo/prover/protocol/ifaces"
+
+// MainLookupCtx stores the compilation context of all the lookup queries
+// altogether.
+type mainLookupCtx struct {
+
+	// LookupTables stores all the lookup table the compiler encounters. They are
+	// sorted in canonical order. This used to derive a determistic ordering
+	// of the lookup LookupTables. (We want to ensure the compiler yields always
+	// exactly the same result for replicability).
+	//
+	// To illustrates its structure, the following sub-statement
+	//
+	// 		table[numTable][frag]
+	//
+	// refers to to the fragment #frag of the the table #numTable.
+	LookupTables [][]table
+
+	// CheckedTables stores all the checked column by lookup table. The key is
+	// obtained as nameTable(lookupTable) where lookup is sorted in
+	// canonical order.
+	CheckedTables map[string][]table
+
+	// IncludedFilters stores all the filters for the checked columns and `nil`
+	// if no filter is applied. As for [checkedTables] they are stored by
+	// lookup table name and in the same order for each key.
+	IncludedFilters map[string][]ifaces.Column
+
+	// Rounds stores the interaction round assigned to each lookupTable. The
+	// round is obtained by taking the max of the declaration Rounds of the
+	// Inclusion queries using the corresponding lookup table.
+	Rounds map[string]int
+
+	// it stores the multiplicity of T is S.
+	//  For the multiColum case it collapse T and S and then counts the multiplicity.
+	// note that here the collapsing does not need the same randomness as the compilation.
+	// since the multiplicity is the same w.r.t any randomness.
+	mTables map[string]table
+}

prover/protocol/distributed/compiler/inclusion/inclusion.go

@@ -23,10 +23,11 @@ import (
 func CompileLogDerivative(moduleComp *wizard.CompiledIOP) {
 
 	var (
+		// capture the S and T columns relevant to the module.
 		mainLookupCtx = captureModuleLookupTables(moduleComp)
 		lastRound     = moduleComp.NumRounds() - 1
 
-		// zCatalog stores a mapping (round, size) into [query.LogDerivativeSumInput].
+		// zCatalog stores a mapping from (round, size) into [query.LogDerivativeSumInput].
 		// it packs the sigma columns from the same (round,size) together.
 		zCatalog = map[[2]int]*query.LogDerivativeSumInput{}
 	)
@@ -44,14 +45,16 @@ func CompileLogDerivative(moduleComp *wizard.CompiledIOP) {
 		var (
 			round           = mainLookupCtx.Rounds[tableName]
 			includedFilters = mainLookupCtx.IncludedFilters[tableName]
-			tableCtx        = collapsMultiColsToSingleCol(moduleComp, round, lookupTable, checkedTables, includedFilters)
+			mTable          = mainLookupCtx.mTables[tableName]
+			// it collapses multiColumn tables to single columns.
+			tableCtx = collapsMultiColToSingleCol(moduleComp, round, lookupTable, checkedTables, includedFilters, mTable)
 		)
 
 		// push to zCatalog
 		PushToZCatalog(tableCtx, zCatalog)
 	}
 
-	// Handle cases where only T part is present
+	// Handle cases where only T part is present in the module
 	for _, lookupTable := range mainLookupCtx.LookupTables {
 		tableName := lookUp.NameTable(lookupTable)
 		if _, ok := mainLookupCtx.CheckedTables[tableName]; ok {
@@ -60,14 +63,15 @@ func CompileLogDerivative(moduleComp *wizard.CompiledIOP) {
 
 		var (
 			round    = mainLookupCtx.Rounds[tableName]
-			tableCtx = collapsMultiColsToSingleCol(moduleComp, round, lookupTable, nil, nil)
+			mTable   = mainLookupCtx.mTables[tableName]
+			tableCtx = collapsMultiColToSingleCol(moduleComp, round, lookupTable, nil, nil, mTable)
 		)
 
 		// push to zCatalog
 		PushToZCatalog(tableCtx, zCatalog)
 	}
 
-	// insert a  LogDerivativeSum for all the Sigma Columns .
+	// insert a  LogDerivativeSum for all the Sigma Columns in the module.
 	moduleComp.InsertLogDerivativeSum(lastRound, "LogDerivativeSum", zCatalog)
 }
 
@@ -81,13 +85,15 @@ func findLookupTableByName(lookupTables [][]table, name string) []table {
 	return nil
 }
 
-// It collapses the tables of MultiColumns to single columns. It also sample the Gamma coin for the rest of the compilation.
-func collapsMultiColsToSingleCol(
+// It collapses the tables of MultiColumns to single columns.
+// It also sample the Gamma coin for the rest of the compilation.
+func collapsMultiColToSingleCol(
 	comp *wizard.CompiledIOP,
 	round int,
 	lookupTable []table,
 	checkedTables []table,
 	includedFilters []ifaces.Column,
+	mTable table,
 ) (ctx lookUp.SingleTableCtx) {
 
 	ctx = lookUp.SingleTableCtx{
@@ -134,6 +140,8 @@ func collapsMultiColsToSingleCol(
 		}
 	}
 
+	ctx.M = mTable
+
 	ctx.Gamma = comp.InsertCoin(
 		round+1,
 		lookUp.DeriveTableName[coin.Name](lookUp.LogDerivativePrefix, lookupTable, "GAMMA"),
@@ -143,8 +151,8 @@ func collapsMultiColsToSingleCol(
 	return ctx
 }
 
-// PushToZCatalog constructs the numerators and denominators for S and T of the
-// stc into zCatalog for their corresponding rounds and size.
+// PushToZCatalog constructs the numerators and denominators for the collapsed S and T
+// into zCatalog, for their corresponding rounds and size.
 func PushToZCatalog(stc lookUp.SingleTableCtx, zCatalog map[[2]int]*query.LogDerivativeSumInput) {
 
 	var (

prover/protocol/distributed/compiler/inclusion/lookup_discoverer.go

@@ -14,16 +14,17 @@ import (
 // compiler as a shorthand to make the code more eye-parseable.
 type table = []ifaces.Column
 
-// captureModuleLookupTables inspects comp and looks for Inclusion queries.
+// captureModuleLookupTables inspects moduleComp and looks for Inclusion queries.
 // It groups the matched queries by lookup table and marks them as ignored.
-// It creates a list of lookupTables and checkedTables present in the module.
+// It creates a list of lookupTables, mTables and checkedTables present in the module.
 //
 // The input is a compiledIOP object that stores the columns relevant to the module (in its Column field)
 // Note that for a lookup query the module may contain only S or T table (and not necessarily both).
-func captureModuleLookupTables(moduleComp *wizard.CompiledIOP) lookUp.MainLookupCtx {
+func captureModuleLookupTables(moduleComp *wizard.CompiledIOP) mainLookupCtx {
 	var (
-		ctx = lookUp.MainLookupCtx{
+		ctx = mainLookupCtx{
 			LookupTables:    [][]table{},
+			mTables:         map[string]table{},
 			CheckedTables:   map[string][]table{},
 			IncludedFilters: map[string][]ifaces.Column{},
 			Rounds:          map[string]int{},
@@ -39,11 +40,8 @@ func captureModuleLookupTables(moduleComp *wizard.CompiledIOP) lookUp.MainLookup
 			continue
 		}
 
-		// Determine if the query is relevant to the module
+		// Determine the part of the query that is relevant to the module
 		relevantPart := determineRelevantPart(lookup, moduleComp.Columns)
-		if relevantPart == "" {
-			continue
-		}
 
 		// This ensures that the lookup query is not used again in the
 		// compilation process. We know that the query was already ignored at
@@ -85,6 +83,7 @@ func captureModuleLookupTables(moduleComp *wizard.CompiledIOP) lookUp.MainLookup
 			ctx.IncludedFilters[tableName] = []ifaces.Column{}
 			ctx.CheckedTables[tableName] = []table{}
 			ctx.LookupTables = [][]table{}
+			ctx.mTables[tableName] = table{}
 			ctx.Rounds[tableName] = 0
 		}
 
@@ -93,12 +92,17 @@ func captureModuleLookupTables(moduleComp *wizard.CompiledIOP) lookUp.MainLookup
 			ctx.IncludedFilters[tableName] = append(ctx.IncludedFilters[tableName], includedFilter)
 			ctx.CheckedTables[tableName] = append(ctx.CheckedTables[tableName], checkedTable)
 			ctx.LookupTables = append(ctx.LookupTables, lookupTable)
+			// get the M table from the moduleComp and add it to the  mTables.
+			ctx.mTables[tableName] = mTable(moduleComp, lookupTable)
+
 		}
 		if relevantPart == "S" {
 			ctx.IncludedFilters[tableName] = append(ctx.IncludedFilters[tableName], includedFilter)
 			ctx.CheckedTables[tableName] = append(ctx.CheckedTables[tableName], checkedTable)
 		} else if relevantPart == "T" {
 			ctx.LookupTables = append(ctx.LookupTables, lookupTable)
+			// get the M table from the moduleComp and add it to the  mTables.
+			ctx.mTables[tableName] = mTable(moduleComp, lookupTable)
 		}
 
 		ctx.Rounds[tableName] = max(ctx.Rounds[tableName], moduleComp.QueriesNoParams.Round(lookup.ID))
@@ -129,7 +133,15 @@ func determineRelevantPart(lookup query.Inclusion, moduleColumns column.Store) s
 	if hasS && !hasT {
 		return "S"
 	} else {
-		panic("the module contains a query that is not relevant to any column")
+		panic("the module contains a query that is not relevant to any module-column")
 	}
 
 }
+
+func mTable(comp *wizard.CompiledIOP, t [][]ifaces.Column) (m []ifaces.Column) {
+	for frag := range t[0] {
+		id := ifaces.ColIDf("%v_%v_%v", lookUp.NameTable(t), "M", frag)
+		m = append(m, comp.Columns.GetHandle(id))
+	}
+	return m
+}