diff --git a/cmd/radiance/car/createcar/cmd-create-car.go b/cmd/radiance/car/createcar/cmd-create-car.go index 3623e59..68e689a 100644 --- a/cmd/radiance/car/createcar/cmd-create-car.go +++ b/cmd/radiance/car/createcar/cmd-create-car.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "sort" "strconv" "strings" "time" @@ -314,10 +315,23 @@ func run(c *cobra.Command, args []string) { } klog.Infof("Finalizing DAG in the CAR file for epoch %d, at path: %s", epoch, finalCARFilepath) - epochCID, slotRecap, err := multi.FinalizeDAG(epoch) + epochCID, slotRecap, gotSlots, err := multi.FinalizeDAG(epoch) if err != nil { panic(err) } + { + calculatedSchedule := removeIf(slots, func(block uint64) bool { + return slotedges.CalcEpochForSlot(block) != epoch + }) + processedSlots := removeIf(gotSlots, func(block uint64) bool { + return slotedges.CalcEpochForSlot(block) != epoch + }) + + _, _, areDifferent := compare(calculatedSchedule, processedSlots) + if areDifferent { + klog.Exitf("The calculated schedule and the processed slots (that were written to the CAR file) are different") + } + } klog.Infof("Root of the DAG (Epoch CID): %s", epochCID) tookCarCreation := time.Since(startedAt) klog.Infof("Done. Completed CAR file generation in %s", tookCarCreation) @@ -533,3 +547,85 @@ func getDirSize(path string) (uint64, error) { }) return size, err } + +func removeIf(slice []uint64, remover func(uint64) bool) []uint64 { + var out []uint64 + for _, item := range slice { + if !remover(item) { + out = append(out, item) + } + } + return out +} + +func uniqueBlocks(blocks []uint64) []uint64 { + // sort, then remove duplicates: + sort.Slice(blocks, func(i, j int) bool { + return blocks[i] < blocks[j] + }) + var out []uint64 + for i := range blocks { + if i == 0 || blocks[i] != blocks[i-1] { + out = append(out, blocks[i]) + } + } + return out +} + +func compare(scheduleList []uint64, carList []uint64) ([]uint64, []uint64, bool) { + scheduleList = uniqueBlocks(scheduleList) + carList = uniqueBlocks(carList) + + hasDiff := false + // blocks in schedule but not in car: + var removed []uint64 + for _, block := range scheduleList { + if !contains(carList, block) { + removed = append(removed, block) + } + } + if len(removed) > 0 { + fmt.Printf("🚫 blocks in %s but not in %s:\n", green("schedule"), red("car")) + for _, block := range removed { + fmt.Println(block) + } + hasDiff = true + } + + // blocks in car but not in schedule: + var added []uint64 + for _, block := range carList { + if !contains(scheduleList, block) { + added = append(added, block) + } + } + if len(added) > 0 { + fmt.Printf("🚫 blocks in %s but not in %s:\n", green("car"), red("schedule")) + for _, block := range added { + fmt.Println(block) + } + hasDiff = true + } + + if !hasDiff { + fmt.Println("✅ No differences between schedule and car") + } + return added, removed, hasDiff +} + +func contains(slots []uint64, slot uint64) bool { + i := SearchUint64(slots, slot) + return i < len(slots) && slots[i] == slot +} + +func SearchUint64(a []uint64, x uint64) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +func red(s string) string { + return "\033[31m" + s + "\033[0m" +} + +func green(s string) string { + return "\033[32m" + s + "\033[0m" +} diff --git a/cmd/radiance/car/createcar/multistage.go b/cmd/radiance/car/createcar/multistage.go index 6d32356..9407612 100644 --- a/cmd/radiance/car/createcar/multistage.go +++ b/cmd/radiance/car/createcar/multistage.go @@ -688,7 +688,7 @@ type MultistageRecap struct { // the CAR file with the root of the DAG (the Epoch object CID). func (cw *Multistage) FinalizeDAG( epoch uint64, -) (datamodel.Link, *MultistageRecap, error) { +) (datamodel.Link, *MultistageRecap, []uint64, error) { { // wait for all slots to be registered klog.Infof("Waiting for all slots to be registered...") @@ -702,7 +702,7 @@ func (cw *Multistage) FinalizeDAG( } allRegistered, err := cw.reg.GetAll() if err != nil { - return nil, nil, fmt.Errorf("failed to get all links: %w", err) + return nil, nil, nil, fmt.Errorf("failed to get all links: %w", err) } allSlots := make([]uint64, 0, len(allRegistered)) for _, slot := range allRegistered { @@ -721,7 +721,7 @@ func (cw *Multistage) FinalizeDAG( klog.Infof("Completing DAG for epoch %d...", epoch) epochRootLink, err := cw.constructEpoch(epoch, schedule) if err != nil { - return nil, nil, fmt.Errorf("failed to construct epoch: %w", err) + return nil, nil, nil, fmt.Errorf("failed to construct epoch: %w", err) } klog.Infof("Completed DAG for epoch %d", epoch) @@ -735,7 +735,7 @@ func (cw *Multistage) FinalizeDAG( klog.Infof("Closing CAR...") err = cw.Close() if err != nil { - return nil, nil, fmt.Errorf("failed to close file: %w", err) + return nil, nil, nil, fmt.Errorf("failed to close file: %w", err) } klog.Infof("Closed CAR for epoch %d", epoch) @@ -748,12 +748,12 @@ func (cw *Multistage) FinalizeDAG( klog.Infof("Replacing root in CAR with CID of epoch %d", epoch) err = cw.replaceRoot(epochCid) if err != nil { - return nil, nil, fmt.Errorf("failed to replace roots in file: %w", err) + return nil, nil, nil, fmt.Errorf("failed to replace roots in file: %w", err) } klog.Infof("Replaced root in CAR with CID of epoch %d", epoch) } - return epochRootLink, slotRecap, err + return epochRootLink, slotRecap, allSlots, err } func (cw *Multistage) NumberOfTransactions() uint64 { diff --git a/pkg/blockstore/schedule.go b/pkg/blockstore/schedule.go index 617ba81..6f00220 100644 --- a/pkg/blockstore/schedule.go +++ b/pkg/blockstore/schedule.go @@ -76,13 +76,13 @@ func (s TraversalSchedule) LastSlot() (uint64, bool) { // Each iterates over each DB in the schedule. func (s TraversalSchedule) Each( ctx context.Context, - f func(dbIndex int, db *WalkHandle, slots []uint64) error, + callback func(dbIndex int, db *WalkHandle, slots []uint64) error, ) error { for dbIndex, db := range s.schedule { if ctx.Err() != nil { return ctx.Err() } - if err := f(dbIndex, db.handle, db.slots); err != nil { + if err := callback(dbIndex, db.handle, db.slots); err != nil { if err == ErrStopIteration { return nil }