From 5da3b408de7ddd4216d2c8a840fe64e78fb135fd Mon Sep 17 00:00:00 2001 From: PengChen Date: Fri, 1 Sep 2017 14:46:48 +0800 Subject: [PATCH] #1314 add ut for trackable gc (#1318) --- Makefile | 24 +- src/master/master_impl.h | 1 - src/master/test/master_impl_test.cc | 32 +- src/master/test/master_test.cc | 29 ++ src/master/test/trackable_gc_test.cc | 560 +++++++++++++++++++++++++++ 5 files changed, 612 insertions(+), 34 deletions(-) create mode 100644 src/master/test/master_test.cc create mode 100644 src/master/test/trackable_gc_test.cc diff --git a/Makefile b/Makefile index 761b145cf..c6eb1d15d 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ LDFLAGS += -rdynamic $(DEPS_LDPATH) $(DEPS_LDFLAGS) -lpthread -lrt -lz -ldl \ -lreadline -lncurses -fPIC SO_LDFLAGS += -rdynamic $(DEPS_LDPATH) $(SO_DEPS_LDFLAGS) -lpthread -lrt -lz -ldl \ -shared -fPIC -Wl,--version-script,so-version-script # hide symbol of thirdparty libs +UTFLAGS += -Dprivate=public PROTO_FILES := $(wildcard src/proto/*.proto) PROTO_OUT_CC := $(PROTO_FILES:.proto=.pb.cc) @@ -46,8 +47,9 @@ MONITOR_SRC := src/monitor/teramo_main.cc MARK_SRC := src/benchmark/mark.cc src/benchmark/mark_main.cc TEST_SRC := src/utils/test/prop_tree_test.cc src/utils/test/tprinter_test.cc \ src/io/test/tablet_io_test.cc src/io/test/tablet_scanner_test.cc \ - src/master/test/master_impl_test.cc src/io/test/load_test.cc \ - src/common/test/thread_pool_test.cc + src/io/test/load_test.cc src/master/test/master_test.cc \ + src/master/test/master_impl_test.cc src/master/test/trackable_gc_test.cc \ + src/common/test/thread_pool_test.cc TEST_OUTPUT := test_output UNITTEST_OUTPUT := $(TEST_OUTPUT)/unittest @@ -71,7 +73,7 @@ HTTP_OBJ := $(HTTP_SRC:.cc=.o) TEST_OBJ := $(TEST_SRC:.cc=.o) ALL_OBJ := $(MASTER_OBJ) $(TABLETNODE_OBJ) $(IO_OBJ) $(SDK_OBJ) $(PROTO_OBJ) \ $(JNI_TERA_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(SERVER_OBJ) $(CLIENT_OBJ) \ - $(TEST_CLIENT_OBJ) $(TERA_C_OBJ) $(MONITOR_OBJ) $(MARK_OBJ) $(TEST_OBJ) \ + $(TEST_CLIENT_OBJ) $(TERA_C_OBJ) $(MONITOR_OBJ) $(MARK_OBJ) \ $(SERVER_WRAPPER_OBJ) LEVELDB_LIB := src/leveldb/libleveldb.a LEVELDB_UTIL := src/leveldb/util/histogram.o src/leveldb/port/port_posix.o @@ -83,8 +85,8 @@ TERA_C_SO = libtera_c.so JNILIBRARY = libjni_tera.so BENCHMARK = tera_bench tera_mark TESTS = prop_tree_test tprinter_test string_util_test tablet_io_test \ - tablet_scanner_test fragment_test progress_bar_test master_impl_test load_test \ - thread_pool_test + tablet_scanner_test fragment_test progress_bar_test master_test load_test \ + thread_pool_test .PHONY: all clean cleanall test @@ -100,7 +102,7 @@ all: $(PROGRAM) $(LIBRARY) $(SOLIBRARY) $(TERA_C_SO) $(JNILIBRARY) $(BENCHMARK) test: $(TESTS) mkdir -p $(UNITTEST_OUTPUT) - cp $(TESTS) $(UNITTEST_OUTPUT) + mv $(TESTS) $(UNITTEST_OUTPUT) $(MAKE) test -C src/leveldb cp src/leveldb/*_test $(UNITTEST_OUTPUT) @@ -111,7 +113,7 @@ check: test sh ./src/sdk/python/checker.sh clean: - rm -rf $(ALL_OBJ) $(PROTO_OUT_CC) $(PROTO_OUT_H) $(TEST_OUTPUT) + rm -rf $(ALL_OBJ) $(TEST_OBJ) $(PROTO_OUT_CC) $(PROTO_OUT_H) $(TEST_OUTPUT) $(MAKE) clean -C src/leveldb rm -rf $(PROGRAM) $(LIBRARY) $(SOLIBRARY) $(TERA_C_SO) $(JNILIBRARY) $(BENCHMARK) $(TESTS) terahttp @@ -193,13 +195,17 @@ tablet_scanner_test: src/sdk/tera.o src/io/test/tablet_scanner_test.o src/tablet $(IO_OBJ) $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB) $(CXX) -o $@ $^ $(LDFLAGS) -master_impl_test: src/master/test/master_impl_test.o src/tera_entry.cc $(MASTER_OBJ) $(IO_OBJ) $(SDK_OBJ) \ - $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB) +master_test: src/master/test/master_test.o src/master/test/master_impl_test.o \ + src/master/test/trackable_gc_test.o src/tera_entry.cc $(MASTER_OBJ) $(IO_OBJ) $(SDK_OBJ) \ + $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB) $(CXX) -o $@ $^ $(LDFLAGS) $(ALL_OBJ): %.o: %.cc $(PROTO_OUT_H) $(CXX) $(CXXFLAGS) -c $< -o $@ +$(TEST_OBJ): %.o: %.cc $(PROTO_OUT_H) + $(CXX) $(CXXFLAGS) $(UTFLAGS) -c $< -o $@ + $(VERSION_SRC): FORCE sh build_version.sh diff --git a/src/master/master_impl.h b/src/master/master_impl.h index 34acfabce..a8959c703 100644 --- a/src/master/master_impl.h +++ b/src/master/master_impl.h @@ -154,7 +154,6 @@ class MasterImpl { std::string ProfilingLog(); private: - friend class MasterImplTest; typedef std::function SnapshotClosure; typedef std::function RollbackClosure; typedef std::function DelSnapshotClosure; diff --git a/src/master/test/master_impl_test.cc b/src/master/test/master_impl_test.cc index 8264fc04a..e9e130e33 100644 --- a/src/master/test/master_impl_test.cc +++ b/src/master/test/master_impl_test.cc @@ -4,9 +4,9 @@ #include -#include -#include -#include +#include "gflags/gflags.h" +#include "glog/logging.h" +#include "gtest/gtest.h" #include "common/base/scoped_ptr.h" #include "master/master_impl.h" @@ -14,18 +14,13 @@ #include "utils/utils_cmd.h" #include "version.h" -DECLARE_string(tera_master_port); -DECLARE_string(log_dir); -DECLARE_bool(tera_zk_enabled); -DECLARE_string(tera_leveldb_env_type); -DECLARE_string(tera_fake_zk_path_prefix); - namespace tera { namespace master { class MasterImplTest : public ::testing::Test, public MasterImpl { public: - MasterImplTest() : merge_enter_phase2(false) {} + MasterImplTest() : merge_enter_phase2(false) { + } void SplitTabletTest() { SplitTabletRequest* request = NULL; @@ -107,7 +102,7 @@ class MasterImplTest : public ::testing::Test, public MasterImpl { LOG(ERROR) << "dummy UnloadTabletAsync..."; } - void MergeTabletBorkenTest() { + void MergeTabletBrokenTest() { TablePtr table(new Table("mergetest")); TabletPtr t1 = MakeTabletPtr("", "a", table); t1->SetStatus(kTableReady); @@ -150,21 +145,10 @@ TEST_F(MasterImplTest, MergeTest) { MergeTabletTest(); } -TEST_F(MasterImplTest, MergeTabletBorkenTest) { - MergeTabletBorkenTest(); +TEST_F(MasterImplTest, MergeTabletBrokenTest) { + MergeTabletBrokenTest(); } } // master } // tera -int main(int argc, char** argv) { - ::google::ParseCommandLineFlags(&argc, &argv, true); - ::google::InitGoogleLogging(argv[0]); - FLAGS_tera_zk_enabled = false; - FLAGS_tera_leveldb_env_type = "local"; - - tera::utils::SetupLog("master_test"); - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - diff --git a/src/master/test/master_test.cc b/src/master/test/master_test.cc new file mode 100644 index 000000000..89b44c208 --- /dev/null +++ b/src/master/test/master_test.cc @@ -0,0 +1,29 @@ +// Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gflags/gflags.h" +#include "glog/logging.h" +#include "gtest/gtest.h" + +#include "utils/utils_cmd.h" + +DECLARE_string(tera_master_port); +DECLARE_string(log_dir); +DECLARE_bool(tera_zk_enabled); +DECLARE_string(tera_leveldb_env_type); +DECLARE_string(tera_fake_zk_path_prefix); + +int main(int argc, char** argv) { + ::google::ParseCommandLineFlags(&argc, &argv, true); + ::google::InitGoogleLogging(argv[0]); + + FLAGS_tera_zk_enabled = false; + FLAGS_tera_leveldb_env_type = "local"; + + tera::utils::SetupLog("master_test"); + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} + diff --git a/src/master/test/trackable_gc_test.cc b/src/master/test/trackable_gc_test.cc new file mode 100644 index 000000000..7d6c78dd6 --- /dev/null +++ b/src/master/test/trackable_gc_test.cc @@ -0,0 +1,560 @@ +// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "gflags/gflags.h" +#include "glog/logging.h" +#include "gtest/gtest.h" + +#include "db/filename.h" +#include "io/utils_leveldb.h" +#include "master/tablet_manager.h" +#include "utils/utils_cmd.h" + +DECLARE_bool(tera_zk_enabled); +DECLARE_string(tera_leveldb_env_type); +DECLARE_string(tera_master_gc_strategy); +DECLARE_string(tera_tabletnode_path_prefix); + +namespace tera { +namespace master { + +class TrackableGcTest : public ::testing::Test { +public: + TrackableGcTest() : mgr_(nullptr, nullptr, nullptr) { + std::cout << "TrackableGcTest()" << std::endl; + } + virtual ~TrackableGcTest() { + std::cout << "~TrackableGcTest()" << std::endl; + } + + TablePtr CreateTable(const std::string& name) { + TablePtr table(new Table(name)); + TableMeta meta; + EXPECT_TRUE(mgr_.AddTable(name, meta, &table, nullptr)); + std::cout << "create table " << name << " success" << std::endl; + + return table; + } + + TabletMeta CreateTabletMeta(const std::string& table_name, const std::string& start, const std::string& end) { + TabletMeta meta; + meta.set_table_name(table_name); + meta.mutable_key_range()->set_key_start(start); + meta.mutable_key_range()->set_key_end(end); + + return meta; + } + + TabletPtr CreateTablet(const std::string& start, const std::string& end, TablePtr table) { + TabletMeta meta = CreateTabletMeta(table->GetTableName(), start, end); + TabletPtr tablet(new Tablet(meta, table)); + TableSchema schema; + EXPECT_TRUE(mgr_.AddTablet(meta, schema, &tablet)); + std::cout << "create tablet [" << start << ", " << end << "]" << " success" << std::endl; + + return tablet; + } + + TabletFile CreateTabletFile(uint64_t tablet_id, uint32_t lg_id, uint64_t file_id, bool create_local_file = false) { + TabletFile tablet_file; + tablet_file.tablet_id = tablet_id; + tablet_file.lg_id = lg_id; + tablet_file.file_id = file_id; + + if (create_local_file) { + leveldb::Env* env = io::LeveldbBaseEnv(); + std::string table_path = FLAGS_tera_tabletnode_path_prefix + kTableName_; + std::string path; + + if (lg_id == 0 && file_id == 0) { + path = leveldb::BuildTabletPath(table_path, tablet_id); + EXPECT_TRUE(env->CreateDir(path).ok()); + } else { + path = leveldb::BuildTableFilePath(table_path, tablet_id, lg_id, file_id); + size_t dir_pos = path.rfind("/"); + EXPECT_TRUE(dir_pos != std::string::npos); + EXPECT_TRUE(env->CreateDir(path.substr(0, dir_pos)).ok()); + leveldb::WritableFile* writable_file; + EXPECT_TRUE(env->NewWritableFile(path, &writable_file).ok()); + delete writable_file; + } + } + + return tablet_file; + } + + TabletInheritedFileInfo CreateTabletInheritedFileInfo(const TabletPtr& tablet, const TabletFile& tablet_file) { + TabletInheritedFileInfo inh_file_info; + inh_file_info.set_table_name(tablet->GetTableName()); + inh_file_info.set_key_start(tablet->GetKeyStart()); + inh_file_info.set_key_end(tablet->GetKeyEnd()); + + LgInheritedLiveFiles* lg_files = inh_file_info.add_lg_inh_files(); + lg_files->set_lg_no(tablet_file.lg_id); + lg_files->add_file_number(1UL << 63 | tablet_file.tablet_id << 32 | tablet_file.file_id); + + return inh_file_info; + } + + TabletInheritedFileInfo CreateEmptyTabletInheritedFileInfo(const TabletPtr& tablet) { + TabletInheritedFileInfo inh_file_info; + inh_file_info.set_table_name(tablet->GetTableName()); + inh_file_info.set_key_start(tablet->GetKeyStart()); + inh_file_info.set_key_end(tablet->GetKeyEnd()); + + return inh_file_info; + } + + void TestAddInheritedFile() { + TablePtr table = CreateTable(kTableName_); + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + + TabletFile file1 = CreateTabletFile(1, 0, 1); + MutexLock l(&table->mutex_); + + // first add, ref inc to 1 + // this step simulates collecting inherited file from filesystem + table->AddInheritedFile(file1, false); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // add the same file again, ref will not inc + // this step simulates collecting inherited file from filesystem again + table->AddInheritedFile(file1, false); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // add the same file again with ref, ref will inc to 2 + // this step simulates ts reporting using the TabletFile + table->AddInheritedFile(file1, true); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(2, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // add a new file from the same tablet, ref will inc to 1 + TabletFile file2 = CreateTabletFile(1, 0, 2); + table->AddInheritedFile(file2, false); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(2, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file2].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // add a new file from a different tablet, ref will inc to 1 + TabletFile file3 = CreateTabletFile(2, 0, 1); + table->AddInheritedFile(file3, false); + ASSERT_EQ(2, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(2, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[2].size()); + ASSERT_EQ(1, table->useful_inh_files_[2][file3].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + } + + void TestReleaseInheritedFile() { + TablePtr table = CreateTable(kTableName_); + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + + TabletFile file1 = CreateTabletFile(1, 0, 1); + MutexLock l(&table->mutex_); + + // first add, ref inc to 1 + // this step simulates collecting inherited file from filesystem + table->AddInheritedFile(file1, false); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // add the same file again with ref, ref will inc to 2 + // this step simulates ts reporting using the TabletFile + table->AddInheritedFile(file1, true); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(2, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // reease the file, ref will dec to 1 + // this step simulates ts reporting unusing the TabletFile + table->ReleaseInheritedFile(file1); + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // this step simulates all live tablets fo the table have reported + table->EnableDeadTabletGarbageCollect(1); + ASSERT_EQ(0, table->useful_inh_files_.size()); + ASSERT_EQ(2, table->obsolete_inh_files_.size()); // tablet dir and file 1-0-1 + } + + void TestSplit() { + TablePtr table = CreateTable(kTableName_); + TabletPtr tablet_1 = CreateTablet("a", "z", table); + + TabletFile file1 = CreateTabletFile(1, 0, 1); + { + MutexLock l(&table->mutex_); + table->AddInheritedFile(file1, false); + } + table->reported_live_tablets_num_ = 1; + tablet_1->inh_files_.insert(file1); + tablet_1->gc_reported_ = true; + + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + TabletPtr tablet_2, tablet_3; + TabletMeta meta_2 = CreateTabletMeta(table->GetTableName(), "a", "k"); + TabletMeta meta_3 = CreateTabletMeta(table->GetTableName(), "k", "z"); + + table->SplitTablet(tablet_1, meta_2, meta_3, &tablet_2, &tablet_3); + + // afer split: + // 1. each sub tablet shoud ref the inh file from the parent tablet + // 2. ref to the file shoud inc to 2 + // 3. reported_live_tablets_num_ shoud dec 1 if the parent tablet has reported + + // 1 + ASSERT_EQ(1, tablet_1->inh_files_.size()); + ASSERT_EQ(1, tablet_2->inh_files_.size()); + ASSERT_EQ(1, tablet_3->inh_files_.size()); + + // 2 + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(2, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // 3 + ASSERT_EQ(0, table->reported_live_tablets_num_); + } + + void TestMerge() { + TablePtr table = CreateTable(kTableName_); + TabletPtr tablet_1 = CreateTablet("a", "k", table); + TabletPtr tablet_2 = CreateTablet("k", "z", table); + + TabletFile file1 = CreateTabletFile(1, 0, 1); + { + MutexLock l(&table->mutex_); + table->AddInheritedFile(file1, false); + } + TabletFile file2 = CreateTabletFile(2, 0, 1); + { + MutexLock l(&table->mutex_); + table->AddInheritedFile(file2, false); + } + + table->reported_live_tablets_num_ = 2; + tablet_1->inh_files_.insert(file1); + tablet_1->gc_reported_ = true; + tablet_2->inh_files_.insert(file2); + tablet_2->gc_reported_ = true; + + ASSERT_EQ(2, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(2, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(1, table->useful_inh_files_[2].size()); + ASSERT_EQ(1, table->useful_inh_files_[2][file2].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + TabletPtr tablet_3; + TabletMeta meta_3 = CreateTabletMeta(table->GetTableName(), "a", "z"); + table->MergeTablets(tablet_1, tablet_2, meta_3, &tablet_3); + + // afer merge: + // 1. the merged tablet shoud ref all the inh file from the two parent tablet + // 2. ref to the file shoud not change + // 3. reported_live_tablets_num_ shoud dec by the reported num of parent tablets + + // 1 + ASSERT_EQ(1, tablet_1->inh_files_.size()); + ASSERT_EQ(1, tablet_2->inh_files_.size()); + ASSERT_EQ(2, tablet_3->inh_files_.size()); + + // 2 + ASSERT_EQ(2, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(2, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(1, table->useful_inh_files_[2].size()); + ASSERT_EQ(1, table->useful_inh_files_[2][file2].ref); + + // 3 + ASSERT_EQ(0, table->reported_live_tablets_num_); + } + + // Case1: ts report using file1 when master restart + void TestGarbageCollect1() { + TablePtr table = CreateTable(kTableName_); + table->reported_live_tablets_num_ = 0; + TabletPtr tablet_1 = CreateTablet("a", "z", table); + TabletFile file1 = CreateTabletFile(2, 0, 1); + + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + ASSERT_EQ(0, tablet_1->inh_files_.size()); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + // no tablets have reported till now + ASSERT_EQ(0, table->reported_live_tablets_num_); + + // this step simulates tablet_1 reporting using file1 + TabletInheritedFileInfo inh_file_1 = CreateTabletInheritedFileInfo(tablet_1, file1); + table->GarbageCollect(inh_file_1); + ASSERT_EQ(1, tablet_1->inh_files_.size()); + + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[2].size()); + // ref inc to 1: + // first, ref inc to 2 since tablet_1 report using file1 with need_ref=TRUE + // then, ref des to 1 since EnableDeadTabletGarbageCollect() + ASSERT_EQ(1, table->useful_inh_files_[2][file1].ref); + + // all live tablets have reported, but file1 hasn't been released, gc won't work + ASSERT_EQ(1, table->reported_live_tablets_num_); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + } + + // Case2: ts report using file1, then report releasing file1 + void TestGarbageCollect2() { + TablePtr table = CreateTable(kTableName_); + table->reported_live_tablets_num_ = 0; + TabletPtr tablet_1 = CreateTablet("a", "z", table); + TabletFile file1 = CreateTabletFile(2, 0, 1); + + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + ASSERT_EQ(0, tablet_1->inh_files_.size()); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + // no tablets have reported till now + ASSERT_EQ(0, table->reported_live_tablets_num_); + + // this step simulates tablet_1 reporting using file1 + TabletInheritedFileInfo inh_file_1 = CreateTabletInheritedFileInfo(tablet_1, file1); + table->GarbageCollect(inh_file_1); + ASSERT_EQ(1, tablet_1->inh_files_.size()); + + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[2].size()); + // ref inc to 1: + // first, ref inc to 2 since tablet_1 report using file1 with need_ref=TRUE + // then, ref des to 1 since EnableDeadTabletGarbageCollect() + ASSERT_EQ(1, table->useful_inh_files_[2][file1].ref); + + // all live tablets have reported, but file1 hasn't been released, gc won't work + ASSERT_EQ(1, table->reported_live_tablets_num_); + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // this step simulates tablet_1 reporting releasing file1 + TabletInheritedFileInfo inh_file_2 = CreateEmptyTabletInheritedFileInfo(tablet_1); + table->GarbageCollect(inh_file_2); + ASSERT_EQ(0, tablet_1->inh_files_.size()); + + // all live tablets have reported, and ref files have been released, gc worked + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + ASSERT_EQ(2, table->obsolete_inh_files_.size()); // tablet dir and file 2-0-1 + } + + // Case3: tablet_1 split into tablet_2 and tablet_3, + // then tablet_2 and tablet_3 will report + void TestGarbageCollect3() { + TablePtr table = CreateTable(kTableName_); + table->reported_live_tablets_num_ = 0; + TabletPtr tablet_1 = CreateTablet("a", "z", table); + TabletFile file1 = CreateTabletFile(1, 0, 1); + { + MutexLock l(&table->mutex_); + table->AddInheritedFile(file1, false); + } + + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // no tablets have reported till now + ASSERT_EQ(0, table->reported_live_tablets_num_); + + TabletPtr tablet_2, tablet_3; + TabletMeta meta_2 = CreateTabletMeta(table->GetTableName(), "a", "k"); + TabletMeta meta_3 = CreateTabletMeta(table->GetTableName(), "k", "z"); + + table->SplitTablet(tablet_1, meta_2, meta_3, &tablet_2, &tablet_3); + + // suppose after split, tablet_2 will ref file1 and talbet_3 has no ref + + // this step simulates tablet_2 reporting using file1 + TabletInheritedFileInfo inh_file_1 = CreateTabletInheritedFileInfo(tablet_2, file1); + table->GarbageCollect(inh_file_1); + ASSERT_EQ(1, tablet_2->inh_files_.size()); + + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + // ref inc to 2 since tablet_2 report using file1 + ASSERT_EQ(2, table->useful_inh_files_[1][file1].ref); + + // only tablet_2 has reported, gc won't work + ASSERT_EQ(1, table->reported_live_tablets_num_); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // this step simulates tablet_3 reporting using no file + TabletInheritedFileInfo inh_file_2 = CreateEmptyTabletInheritedFileInfo(tablet_3); + table->GarbageCollect(inh_file_2); + ASSERT_EQ(0, tablet_3->inh_files_.size()); + + // all live tablets have reported, but file1 hasn't been released, gc won't work + ASSERT_EQ(2, table->reported_live_tablets_num_); + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // this step simulates tablet_2 reporting releasing file1 + TabletInheritedFileInfo inh_file_3 = CreateEmptyTabletInheritedFileInfo(tablet_2); + table->GarbageCollect(inh_file_3); + ASSERT_EQ(0, tablet_2->inh_files_.size()); + + // all live tablets have reported, and ref files have been released, gc worked + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + ASSERT_EQ(2, table->obsolete_inh_files_.size()); // tablet dir and file 1-0-1 + } + + void TestCleanObsoleteFile() { + TablePtr table = CreateTable(kTableName_); + + // empty + ASSERT_EQ(0, table->CleanObsoleteFile()); + + // add a TabletFile and tablet dir + TabletFile file1 = CreateTabletFile(1, 0, 1, true); + TabletFile tablet_dir = CreateTabletFile(1, 0, 0, true); + table->obsolete_inh_files_.push(file1); + table->obsolete_inh_files_.push(tablet_dir); + + // obsolete_inh_files_ is has tablet_dir and file1 + ASSERT_EQ(2, table->obsolete_inh_files_.size()); + + // successfully delete 2 files: tablet_dir and file1 + ASSERT_EQ(2, table->CleanObsoleteFile()); + + // obsolete_inh_files_ is empty now + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + } + + void TestTryCollectInheritedFile() { + TablePtr table = CreateTable(kTableName_); + + // no dead tablet + ASSERT_FALSE(table->TryCollectInheritedFile()); + + // dead tablet1 with file1 + TabletFile file1 = CreateTabletFile(1, 0, 1, true); + + ASSERT_EQ(0, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(0, table->useful_inh_files_.size()); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + + // collect dead tablet from filesystem + ASSERT_TRUE(table->TryCollectInheritedFile()); + + ASSERT_EQ(1, table->gc_disabled_dead_tablets_.size()); + ASSERT_EQ(1, table->useful_inh_files_.size()); + ASSERT_EQ(1, table->useful_inh_files_[1].size()); + ASSERT_EQ(1, table->useful_inh_files_[1][file1].ref); + ASSERT_EQ(0, table->obsolete_inh_files_.size()); + } + +protected: + virtual void SetUp() { + std::cout << "SetUp" << std::endl; + } + + virtual void TearDown() { + mgr_.ClearTableList(); + std::cout << "TearDown" << std::endl; + } + + static void SetUpTestCase() { + std::cout << "SetUpTestCase" << std::endl; + FLAGS_tera_zk_enabled = false; + FLAGS_tera_leveldb_env_type = "local"; + FLAGS_tera_master_gc_strategy = "trackable"; + FLAGS_tera_tabletnode_path_prefix = "./"; + } + + static void TearDownTestCase() { + std::cout << "TearDownTestCase" << std::endl; + std::string table_path = FLAGS_tera_tabletnode_path_prefix + kTableName_; + EXPECT_TRUE(io::DeleteEnvDir(table_path).ok()); + } + +private: + TabletManager mgr_; + const static std::string kTableName_; +}; + +const std::string TrackableGcTest::kTableName_ = "MasterTestTable"; + +TEST_F(TrackableGcTest, AddInheritedFile) { + TestAddInheritedFile(); +} + +TEST_F(TrackableGcTest, ReleaseInheritedFile) { + TestReleaseInheritedFile(); +} + +TEST_F(TrackableGcTest, Split) { + TestSplit(); +} + +TEST_F(TrackableGcTest, Merge) { + TestMerge(); +} + +TEST_F(TrackableGcTest, GarbageCollect1) { + TestGarbageCollect1(); +} + +TEST_F(TrackableGcTest, GarbageCollect2) { + TestGarbageCollect2(); +} + +TEST_F(TrackableGcTest, GarbageCollect3) { + TestGarbageCollect3(); +} + +TEST_F(TrackableGcTest, CleanObsoleteFile) { + TestCleanObsoleteFile(); +} + +TEST_F(TrackableGcTest, TryCollectInheritedFile) { + TestTryCollectInheritedFile(); +} + +} // master +} // tera +