build(deps): bump github.com/cockroachdb/pebble from v0.0.0-202307242…

…34444-7ef7553fd9e1 to v0.0.0-20231114155657-6682fd5e8763 7a4ed3c5 tool: add remotecat dump command ad744ac3 db: keep up to one memtable for recycling 56bba2ac internal/cache: reduce mutex contention from EvictFile ce43e653 internal/cache: increase shard count to 4x CPUs 5e7f8852 db: introduce Download API 9c487492 *: simplify pointCollapsingIter, use vanilla iters for foreign SSTs 42d81d23 internal/rangekey: remove ForeignSSTTransformer fe0ea048 manifest: Added Virtual SSTable Metadata Bounds Checking 83c9361c pebble: Collect Pebble Key Statistics 7e548be8 db: log flushed memtables in-memory size ff9402f0 Makefile: add testcoverage target 077c3e8b ci: use any 1.20 go version 40d3f411 pebble: expose the secondary cache size as an option af8c5f27 internal/cache: mark panic messages as redaction-safe ff5c929a Rate Limit Scan Statistics df7e2ae1 vfs: deflake TestDiskHealthChecking_Filesystem fffe02a1 db: simplify ScanInternal() c055f188 github: code coverage workflows 71f4d6c6 workflows: fix trim-prefix in code-cover-gen.yaml fd0cd8db workflows: fix trim-prefix in code-cover-gen.yaml 49821f58 db: use ordinary ingestion seqnums for shared sstables 58a05cd4 ingest: add ability to ingest external (not local or shared) files f9f63ef2 crossversion: add more comments 18e6ad42 pebble: Export keyspan.Fragmenter 77e81e80 pebble: Update Tokenbucket package and use WaitCtx eb4a8571 *: implement EventuallyFileOnlySnapshots ad242264 *: add error return value to NewIter{,WithContext} 8801607d db: Add TestConcurrentExcise 97285aea sstable: fix two-level monotonic bounds bug 692c3d86 arenaskl: disallow arenas larger than 4GB d77c578f vfs: always swap and verify in health checking updates 856ed364 sstable: fix testing bug that omitted Pebblev4 table format 564b0688 tool: Find command to output invalid SSTable Files 40c977cb replay: Deflake TestCompactionsQuiesce by waiting for sstable copy 1798fbf5 sharedcache: Deflake TestSharedCacheRandomized 8638640b sharedcache: introduce metrics 9bb0864b snapshot: Prevent indefinite wait on compact sync.Cond in EFOS 21f68df4 db: prevent unnecessary memory allocate in batch init 3e9abad7 db: implement newBatchWithSize APIs 1253033c internal/keyspan: refactor interleaving iterator 0b401ee5 workflows: restrict coverage testing to changed packages 8ee73c61 sstable: reduce test memory usage 74d4bb64 sharedcache: reduce random test sizes 1a45921a sstable: use simple user keys in BoundLimitedBlockPropertyFilter 12bbc249 build(deps): bump gopkg.in/yaml.v3 584f7676 objstorageprovider: fix race during sharedSync 7d98cec1 remoteobjcat: improve ApplyBatch code c872ea07 metamorphic: add support for EventuallyFileOnlySnapshots 14df16a6 go.mod: downgrade golang.org/x/exp a207649c metamorphic: allow coverage instrumentation a4fe0500 scripts: add code coverage script 97a33a41 db: lift some helper funcs off of the compactionPicker 0f69e1e3 db: move diskAvailBytes onto compactionEnv 9672ffea manifest: track total size within LevelMetadata 2c133248 db: move manual compaction picking off the compaction picker f94905ad db: remove unused compactionPicker.getEstimatedMaxWAmp 87f4f125 db: crash on logNum invariant violation 556a12ed Use SeekPrefixGE to avoid unnecessary disk read in Get function ec854eea db: rename commitQueue.dequeue to dequeueApplied fd5e2099 db: add documentation for memTable.writerUnref d64de4aa db: add comments on flushable method implementations a3375e8a testkeys: use int64 indexes for KeySpace 7ffe7733 manifest: grab manifest lock when calling InitCompactingFileInfo 0cbd3b1b *: support 32-bit 5283f24b tool: better errors in case of encrypted store 9d4f63d9 db: minor improvements to Options comments 4e6810e2 compaction: allow for multilevel compaction based on WriteAmp heuristic 2129a6e9 compaction: maintain a ref of current version in runCompaction 6f6852d5 event: remove infinite recursion in compactionAnnotations.SafeFormat 4264a7ce tool: fix testdata generator 161df8d9 scripts: code-coverage-publish script 5093058d workflows: add nightly coverage workflow d69d1a01 github: update nightly-code-cover.yaml 7fd73255 metamorphic: fix testkeys.RandomSeparator 1efa535d README.md: add coverage link 3db8edc3 remoteobjcat: move assertions behind invariants.Enabled 20e07e1f db: avoid type conversion during iterator read sampling 96978427 db: add explicit levelIter pointer 2689f0d2 internal/invalidating: move invalidating iter 9dbff72c internal/invalidating: trash trailers with 0xff bb9d6ab6 internal/invalidating: propagate lazy value fetching 529d256a db: use invalidating iterators with low probability under invariants 22fbb69a scripts: generate code coverage with invariants build tag c91e8796 db: double check file reference counts when loading file 336c9979 metamorphic: vary sstable compression algorithm 6d6570bf vfs: enforce mutual exclusion in MemFS.Lock be158640 manifest: remove invalid comment 61f84632 metamorphic: 0 suffix should be suffixless in the RandomSeparator 75bc026c db: add invariant for ValueAndErr 57e248f9 open: Ratchet nextFileNum ahead of objprovider objects 9f260419 pebble: export bucketing scheme used for secondary cache f509cd48 db: only update compaction stats if the compaction succeeds 7d98bc36 ingest: Set bounds with zero sequence number internal keys 244f2e1f tool/logs: add support for flushable ingestions 8f2901cd db: get rid of readstate in EFOS 19bac41e db: move l0SublevelInfo onto compactionLevel 5ac7f5e6 db: refactor construction of compaction input iterator bb005044 db: bump FormatNewest to internalFormatNewest 1d8ff0ba metamorphic: add variant that constructs many versions d038189d db: support virtual sstables in block checksum validation 86593692 db: remove 'Experimental' prefix from recent format major versions a48ac0f4 db: use uncompensated scores to prioritize levels for compaction 895ffed5 db: add SkipPoint iterator option 66045002 sharedcache: tolerate small sizes 1d30adcd db: don't ignore range key bounds during excise 14b8ccdc db: add ingest-time splitting of ssts into virtual ones 4df2ce80 metamorphic: Add testing for ingest splits 75888967 Revert "db: add SkipPoint iterator option" 4e353e51 metamorphic: fix block property collectors integration 17c625cf db: Fix FormatMajorVersion check in version_set for VSSTs 5500da0a .*: support async stats for virtual sstables 02b87adf db: improvements to data_test.go da739ee7 db: adds tests for virtual sstable async table stats calculation 3a1391d6 metamorphic: Ignore ErrCancelledCompaction in background errors b5677d86 db: propagate Comparer into levelIter d925b88b db: add SkipPoint iterator option 725ebe29 db: fix excise test 1d696c79 db: cleanup btree obsoletion logic b2da10c6 db: remove trailing whitespace from compacting log line 699fc0e8 db: only create one CreatedBackingTables entry per sstable ccb9a7dc manifest: add invariant check for duplicate file backings aa077af6 go.mod: specify Go 1.20 0f80e184 Update index.html 8317cf38 db: incrementally keep tracking of backing table size cb4dab66 db: add metrics for num backing sstables and size 3c778710 db: add test for virtual sstable checkpointing a05b0192 db: keep track of virtual sstable count in metrics c7c47d6b db: turn testingAlwaysWaitForCleanup into an option 62251e69 db: make checkpoint test even more deterministic b013ca78 db: keep track of virtual sstable size sum d4bf20c5 *: implement local to remote/shared copy compaction a43e418f db: Use the filenum in the Filebacking for lookup 488f557d metamorphic: fix unbounded snapshot access bb2319bb db: get rid of score parameter in compaction 860605c3 go.mod,.github: upgrade to Go 1.21 790e49c9 db: move l0SublevelInfo to the compactionLevel struct 1b8771e1 go.mod: upgrade cockroachdb/redact to v1.1.5 4280b457 internal/base: never redact FileNum or DiskFileNum 470a93d0 db: convert adjustedOutputLevel into a function d0ac26c3 metamorphic: use require.Equal in TestParserRandom 803507a7 go.mod: upgrade cockroachdb/errors to v1.11.1 a678d096 go.mod: revert change to require go 1.21 5ddf67cc db: deflake TestIngestLoadRand f79ae812 db: avoid use of physical filesystem in TestIngestValidation d57a0be4 db: tolerate errors during async ingest sstable validation e3cbe650 db: use DiskFileName for manifest files and memtables 36b3f57b db: return structured errors from checkConsistency ede31f1a event: mark `FormatMajorVersion` as safe from redaction bbbf3df1 objstorage: fix up redaction, error message formatting 8c13366c compaction: remove the duplicated function mergeFileInfo 94ccf353 vfs/errorfs: add facilities for error injection in datadriven tests 5807b591 use the constant manifest.NumLevels instead of literal value 6205042b compaction: remove the useless condition in check delete compactions f6cde3fc internal/manual,internal/rawalloc: Reduce max array size to support MIPS. 593a72bf db: add NewIterWithContext to Reader dda4d0d2 vfs/errorfs: add OpFileReadAt predicate babd592d db: defer loading L0 range key blocks during iterator construction ec8d19ff vfs/errorfs: add Randomly 38191218 db: retry manual compactions that get cancelled 422dce91 db: add Errorf to Logger interface f522e2f9 build: use go1.21 ed45a776 db: add Batch.NewBatchOnlyIter that reads only from the Batch d4881882 *: use Go 1.21 new primitives: min, max, clear b9263c96 *: use new slices, cmp standard library funcs 33a77e17 internal/keyspan: fix InterleavingIter error handling a9906157 metamorphic: Add TestMetaTwoInstance df07a1e1 vfs: pass start time in nanos to disk-health timing functions 5f131c12 db: default to zeroing fields in Batch.Reset 424019fb db: guard against concurrent batch writes during Commit 86ec1c41 db: move size check for remote files to table stats job 481da041 db: add KeyStatistics.LatestKindsCount 58cbdb46 internal/dsl: new package 844f0582 tool: fix tool to tolerate T in the log context 369800c6 metamorphic: allow snapshots in multi-instance, remove `dbObjID` b4261546 internal/keyspan: consolidate datadriven iterator ops 9a4379bb internal/keyspan: fix defragmenting iterator error handling a0b01b62 db: add SetContext method to Iterator to override existing context 81d9a4fe db: fix nil pointer dereference in SetContext 8574e513 internal/keyspan: obey Seek invariants in filteringIter 96a8bc2d internal/keyspan: document InterleavingIter error invariants 4a0b03c6 metamorphic: ingest range keys dfe1b00a internal/rangekey: fix invariant check with onlySets=false 11da8760 internal/keyspan: test MergingIter error handling b224e8b9 internal/keyspan: fix merge skew a54c529a Makefile: don't git fetch for all targets a515dcb5 sstable: remove valueBlockReader.ctx field c7f6b98a db: categorized stats for sstable iterators dcfb84cc metamorphic: fix range key ingestion c0b4bd44 metamorphic: sync on derived dbs in ingestOp 6682fd5e itertest: add internal iterator datadriven helper
kakao · Dec 28, 2023 · 424eee0 · 424eee0
1 parent 0cb6bbf
commit 424eee0
Show file tree

Hide file tree

Showing 260 changed files with 43,242 additions and 8,197 deletions.
diff --git a/go.mod b/go.mod
@@ -3,7 +3,7 @@ module github.com/kakao/varlog
 go 1.21
 
 require (
-	github.com/cockroachdb/pebble v0.0.0-20230724234444-7ef7553fd9e1
+	github.com/cockroachdb/pebble v0.0.0-20231114155657-6682fd5e8763
 	github.com/docker/go-units v0.5.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/gogo/status v1.1.1
@@ -30,7 +30,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.26.0
-	golang.org/x/exp v0.0.0-20221114191408-850992195362
+	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5
 	golang.org/x/sync v0.5.0
 	golang.org/x/sys v0.15.0
@@ -48,25 +48,25 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cockroachdb/errors v1.8.1 // indirect
-	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
-	github.com/cockroachdb/redact v1.0.8 // indirect
-	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 // indirect
+	github.com/cockroachdb/errors v1.11.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/coreos/go-semver v0.2.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 // indirect
 	github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
-	github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a // indirect
+	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect

diff --git a/go.sum b/go.sum
diff --git a/internal/storage/logger.go b/internal/storage/logger.go
@@ -21,6 +21,10 @@ func (l *logAdaptor) Infof(format string, args ...interface{}) {
 	l.logger.Infof(format, args...)
 }
 
+func (l *logAdaptor) Errorf(format string, args ...interface{}) {
+	l.logger.Errorf(format, args...)
+}
+
 func (l *logAdaptor) Fatalf(format string, args ...interface{}) {
 	l.logger.Fatalf(format, args...)
 }
diff --git a/internal/storage/recovery_points.go b/internal/storage/recovery_points.go
@@ -35,7 +35,10 @@ func (s *Storage) ReadRecoveryPoints() (rp RecoveryPoints, err error) {
 	if err != nil {
 		return
 	}
-	rp.CommittedLogEntry.First, rp.CommittedLogEntry.Last = s.readLogEntryBoundaries()
+	rp.CommittedLogEntry.First, rp.CommittedLogEntry.Last, err = s.readLogEntryBoundaries()
+	if err != nil {
+		return
+	}
 
 	uncommittedBegin := types.MinLLSN
 	if cc := rp.LastCommitContext; cc != nil {
@@ -62,31 +65,39 @@ func (s *Storage) readLastCommitContext() (*CommitContext, error) {
 	return &cc, nil
 }
 
-func (s *Storage) readLogEntryBoundaries() (first, last *varlogpb.LogSequenceNumber) {
-	dit := s.dataDB.NewIter(&pebble.IterOptions{
+func (s *Storage) readLogEntryBoundaries() (first, last *varlogpb.LogSequenceNumber, err error) {
+	dit, err := s.dataDB.NewIter(&pebble.IterOptions{
 		LowerBound: []byte{dataKeyPrefix},
 		UpperBound: []byte{dataKeySentinelPrefix},
 	})
-	cit := s.commitDB.NewIter(&pebble.IterOptions{
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = dit.Close()
+	}()
+	cit, err := s.commitDB.NewIter(&pebble.IterOptions{
 		LowerBound: []byte{commitKeyPrefix},
 		UpperBound: []byte{commitKeySentinelPrefix},
 	})
+	if err != nil {
+		return
+	}
 	defer func() {
-		_ = dit.Close()
 		_ = cit.Close()
 	}()
 
 	first = s.getFirstLogSequenceNumber(cit, dit)
 	if first == nil {
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	last = s.getLastLogSequenceNumber(cit, dit, first)
 	if last == nil {
 		s.logger.Warn("the last must exist but could not be found.", zap.Stringer("first", first))
-		return nil, nil
+		return nil, nil, nil
 	}
-	return first, last
+	return first, last, nil
 }
 
 func (s *Storage) getFirstLogSequenceNumber(cit, dit *pebble.Iterator) *varlogpb.LogSequenceNumber {
@@ -173,10 +184,13 @@ func (s *Storage) getLastLogSequenceNumber(cit, dit *pebble.Iterator, first *var
 func (s *Storage) readUncommittedLogEntryBoundaries(uncommittedBegin types.LLSN) (begin, end types.LLSN, err error) {
 	dk := make([]byte, dataKeyLength)
 	dk = encodeDataKeyInternal(uncommittedBegin, dk)
-	it := s.dataDB.NewIter(&pebble.IterOptions{
+	it, err := s.dataDB.NewIter(&pebble.IterOptions{
 		LowerBound: dk,
 		UpperBound: []byte{dataKeySentinelPrefix},
 	})
+	if err != nil {
+		return types.InvalidLLSN, types.InvalidLLSN, err
+	}
 	defer func() {
 		_ = it.Close()
 	}()

diff --git a/internal/storage/scanner.go b/internal/storage/scanner.go
@@ -59,8 +59,10 @@ func (s *Scanner) Next() bool {
 	return s.it.Next()
 }
 
-func (s *Scanner) Close() error {
-	err := s.it.Close()
+func (s *Scanner) Close() (err error) {
+	if s.it != nil {
+		err = s.it.Close()
+	}
 	s.release()
 	return err
 }

diff --git a/internal/storage/storage.go b/internal/storage/storage.go
@@ -107,7 +107,7 @@ func (s *Storage) newDB(path string, cfg *dbConfig) (*pebble.DB, error) {
 		L0StopWritesThreshold:       cfg.l0StopWritesThreshold,
 		LBaseMaxBytes:               cfg.lbaseMaxBytes,
 		MaxOpenFiles:                cfg.maxOpenFiles,
-		MemTableSize:                cfg.memTableSize,
+		MemTableSize:                uint64(cfg.memTableSize),
 		MemTableStopWritesThreshold: cfg.memTableStopWritesThreshold,
 		MaxConcurrentCompactions:    func() int { return cfg.maxConcurrentCompaction },
 		Levels:                      make([]pebble.LevelOptions, 7),
@@ -181,22 +181,26 @@ func (s *Storage) NewAppendBatch() *AppendBatch {
 }
 
 // NewScanner creates a scanner for the given key range.
-func (s *Storage) NewScanner(opts ...ScanOption) *Scanner {
-	scanner := newScanner()
+func (s *Storage) NewScanner(opts ...ScanOption) (scanner *Scanner, err error) {
+	scanner = newScanner()
 	scanner.scanConfig = newScanConfig(opts)
 	scanner.stg = s
 	itOpt := &pebble.IterOptions{}
 	if scanner.withGLSN {
 		itOpt.LowerBound = encodeCommitKeyInternal(scanner.begin.GLSN, scanner.cks.lower)
 		itOpt.UpperBound = encodeCommitKeyInternal(scanner.end.GLSN, scanner.cks.upper)
-		scanner.it = s.commitDB.NewIter(itOpt)
+		scanner.it, err = s.commitDB.NewIter(itOpt)
 	} else {
 		itOpt.LowerBound = encodeDataKeyInternal(scanner.begin.LLSN, scanner.dks.lower)
 		itOpt.UpperBound = encodeDataKeyInternal(scanner.end.LLSN, scanner.dks.upper)
-		scanner.it = s.dataDB.NewIter(itOpt)
+		scanner.it, err = s.dataDB.NewIter(itOpt)
+	}
+	if err != nil {
+		_ = scanner.Close()
+		return nil, err
 	}
 	_ = scanner.it.First()
-	return scanner
+	return scanner, nil
 }
 
 // Read reads the log entry at the glsn.
@@ -209,7 +213,10 @@ func (s *Storage) Read(opts ...ReadOption) (le varlogpb.LogEntry, err error) {
 }
 
 func (s *Storage) readGLSN(glsn types.GLSN) (le varlogpb.LogEntry, err error) {
-	scanner := s.NewScanner(WithGLSN(glsn, glsn+1))
+	scanner, err := s.NewScanner(WithGLSN(glsn, glsn+1))
+	if err != nil {
+		return
+	}
 	defer func() {
 		_ = scanner.Close()
 	}()
@@ -220,10 +227,13 @@ func (s *Storage) readGLSN(glsn types.GLSN) (le varlogpb.LogEntry, err error) {
 }
 
 func (s *Storage) readLLSN(llsn types.LLSN) (le varlogpb.LogEntry, err error) {
-	it := s.commitDB.NewIter(&pebble.IterOptions{
+	it, err := s.commitDB.NewIter(&pebble.IterOptions{
 		LowerBound: []byte{commitKeyPrefix},
 		UpperBound: []byte{commitKeySentinelPrefix},
 	})
+	if err != nil {
+		return
+	}
 	defer func() {
 		_ = it.Close()
 	}()
@@ -307,10 +317,13 @@ func (s *Storage) findLTE(glsn types.GLSN) (lem varlogpb.LogEntryMeta, err error
 		upper = []byte{commitKeySentinelPrefix}
 	}
 
-	it := s.commitDB.NewIter(&pebble.IterOptions{
+	it, err := s.commitDB.NewIter(&pebble.IterOptions{
 		LowerBound: []byte{commitKeyPrefix},
 		UpperBound: upper,
 	})
+	if err != nil {
+		return
+	}
 	defer func() {
 		_ = it.Close()
 	}()