Skip to content

Commit

Permalink
Add tests for loss-guided expand (#62)
Browse files Browse the repository at this point in the history 
* minor refactoring; adding tests

* remove dead comments

---------

Co-authored-by: Dmitry Razdoburdin <>
  • Loading branch information
@razdoburdin
razdoburdin authored Aug 9, 2024
1 parent bcb4472 commit 3d4af4f
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 22 deletions.
22 changes: 8 additions & 14 deletions plugin/sycl/tree/hist_updater.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,12 @@ void HistUpdater<GradientSumT>::BuildLocalHistograms(
template<typename GradientSumT>
void HistUpdater<GradientSumT>::BuildNodeStats(
const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
builder_monitor_.Start("BuildNodeStats");
for (auto const& entry : qexpand_depth_wise_) {
int nid = entry.nid;
this->InitNewNode(nid, gmat, gpair, *p_fmat, *p_tree);
this->InitNewNode(nid, gmat, gpair, *p_tree);
// add constraints
if (!(*p_tree)[nid].IsLeftChild() && !(*p_tree)[nid].IsRoot()) {
// it's a right child
Expand Down Expand Up @@ -232,7 +231,6 @@ void HistUpdater<GradientSumT>::SplitSiblings(
template<typename GradientSumT>
void HistUpdater<GradientSumT>::ExpandWithDepthWise(
const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
int num_leaves = 0;
Expand All @@ -249,7 +247,7 @@ void HistUpdater<GradientSumT>::ExpandWithDepthWise(
hist_rows_adder_->AddHistRows(this, &sync_ids, p_tree);
BuildLocalHistograms(gmat, p_tree, gpair);
hist_synchronizer_->SyncHistograms(this, sync_ids, p_tree);
BuildNodeStats(gmat, p_fmat, p_tree, gpair);
BuildNodeStats(gmat, p_tree, gpair);

EvaluateAndApplySplits(gmat, p_tree, &num_leaves, depth,
&temp_qexpand_depth);
Expand All @@ -270,7 +268,6 @@ void HistUpdater<GradientSumT>::ExpandWithDepthWise(
template<typename GradientSumT>
void HistUpdater<GradientSumT>::ExpandWithLossGuide(
const common::GHistIndexMatrix& gmat,
DMatrix* p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
builder_monitor_.Start("ExpandWithLossGuide");
Expand All @@ -280,10 +277,10 @@ void HistUpdater<GradientSumT>::ExpandWithLossGuide(
ExpandEntry node(ExpandEntry::kRootNid, p_tree->GetDepth(ExpandEntry::kRootNid));
BuildHistogramsLossGuide(node, gmat, p_tree, gpair);

this->InitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_fmat, *p_tree);

this->InitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_tree);
this->EvaluateSplits({node}, gmat, *p_tree);
node.split.loss_chg = snode_host_[ExpandEntry::kRootNid].best.loss_chg;
// LOG(FATAL) << node.split.loss_chg;

qexpand_loss_guided_->push(node);
++num_leaves;
Expand All @@ -305,9 +302,7 @@ void HistUpdater<GradientSumT>::ExpandWithLossGuide(
e.best.DefaultLeft(), e.weight, left_leaf_weight,
right_leaf_weight, e.best.loss_chg, e.stats.GetHess(),
e.best.left_sum.GetHess(), e.best.right_sum.GetHess());

this->ApplySplit({candidate}, gmat, p_tree);

const int cleft = (*p_tree)[nid].LeftChild();
const int cright = (*p_tree)[nid].RightChild();

Expand All @@ -320,8 +315,8 @@ void HistUpdater<GradientSumT>::ExpandWithLossGuide(
BuildHistogramsLossGuide(right_node, gmat, p_tree, gpair);
}

this->InitNewNode(cleft, gmat, gpair, *p_fmat, *p_tree);
this->InitNewNode(cright, gmat, gpair, *p_fmat, *p_tree);
this->InitNewNode(cleft, gmat, gpair, *p_tree);
this->InitNewNode(cright, gmat, gpair, *p_tree);
bst_uint featureid = snode_host_[nid].best.SplitIndex();
tree_evaluator_.AddSplit(nid, cleft, cright, featureid,
snode_host_[cleft].weight, snode_host_[cright].weight);
Expand Down Expand Up @@ -356,9 +351,9 @@ void HistUpdater<GradientSumT>::Update(

this->InitData(gmat, gpair_device, *p_fmat, *p_tree);
if (param_.grow_policy == xgboost::tree::TrainParam::kLossGuide) {
ExpandWithLossGuide(gmat, p_fmat, p_tree, gpair_device);
ExpandWithLossGuide(gmat, p_tree, gpair_device);
} else {
ExpandWithDepthWise(gmat, p_fmat, p_tree, gpair_device);
ExpandWithDepthWise(gmat, p_tree, gpair_device);
}

for (int nid = 0; nid < p_tree->NumNodes(); ++nid) {
Expand Down Expand Up @@ -842,7 +837,6 @@ void HistUpdater<GradientSumT>::InitNewNode(int nid,
const common::GHistIndexMatrix& gmat,
const USMVector<GradientPair,
MemoryType::on_device> &gpair,
const DMatrix& fmat,
const RegTree& tree) {
builder_monitor_.Start("InitNewNode");

Expand Down
4 changes: 0 additions & 4 deletions plugin/sycl/tree/hist_updater.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,9 @@ class HistUpdater {
void InitNewNode(int nid,
const common::GHistIndexMatrix& gmat,
const USMVector<GradientPair, MemoryType::on_device> &gpair,
const DMatrix& fmat,
const RegTree& tree);

void ExpandWithDepthWise(const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);

Expand All @@ -169,7 +167,6 @@ class HistUpdater {
RegTree *p_tree);

void BuildNodeStats(const common::GHistIndexMatrix &gmat,
DMatrix *p_fmat,
RegTree *p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair);

Expand All @@ -188,7 +185,6 @@ class HistUpdater {
std::vector<ExpandEntry>* temp_qexpand_depth);

void ExpandWithLossGuide(const common::GHistIndexMatrix& gmat,
DMatrix* p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device>& gpair);

Expand Down
70 changes: 66 additions & 4 deletions tests/cpp/plugin/test_sycl_hist_updater.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,8 @@ class TestHistUpdater : public HistUpdater<GradientSumT> {
auto TestInitNewNode(int nid,
const common::GHistIndexMatrix& gmat,
const USMVector<GradientPair, MemoryType::on_device> &gpair,
const DMatrix& fmat,
const RegTree& tree) {
HistUpdater<GradientSumT>::InitNewNode(nid, gmat, gpair, fmat, tree);
HistUpdater<GradientSumT>::InitNewNode(nid, gmat, gpair, tree);
return HistUpdater<GradientSumT>::snode_host_[nid];
}

Expand All @@ -67,6 +66,13 @@ class TestHistUpdater : public HistUpdater<GradientSumT> {
RegTree* p_tree) {
HistUpdater<GradientSumT>::ApplySplit(nodes, gmat, p_tree);
}

auto TestExpandWithLossGuide(const common::GHistIndexMatrix& gmat,
DMatrix *p_fmat,
RegTree* p_tree,
const USMVector<GradientPair, MemoryType::on_device> &gpair) {
HistUpdater<GradientSumT>::ExpandWithLossGuide(gmat, p_tree, gpair);
}
};

void GenerateRandomGPairs(::sycl::queue* qu, GradientPair* gpair_ptr, size_t num_rows, bool has_neg_hess) {
Expand Down Expand Up @@ -295,7 +301,7 @@ void TestHistUpdaterInitNewNode(const xgboost::tree::TrainParam& param, float sp
auto& row_idxs = row_set_collection->Data();
const size_t* row_idxs_ptr = row_idxs.DataConst();
updater.TestBuildHistogramsLossGuide(node, gmat, &tree, gpair);
const auto snode = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_fmat, tree);
const auto snode = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, tree);

GradStats<GradientSumT> grad_stat;
{
Expand Down Expand Up @@ -354,7 +360,7 @@ void TestHistUpdaterEvaluateSplits(const xgboost::tree::TrainParam& param) {
auto& row_idxs = row_set_collection->Data();
const size_t* row_idxs_ptr = row_idxs.DataConst();
const auto* hist = updater.TestBuildHistogramsLossGuide(node, gmat, &tree, gpair);
const auto snode_init = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, *p_fmat, tree);
const auto snode_init = updater.TestInitNewNode(ExpandEntry::kRootNid, gmat, gpair, tree);

const auto snode_updated = updater.TestEvaluateSplits({node}, gmat, tree);
auto best_loss_chg = snode_updated[0].best.loss_chg;
Expand Down Expand Up @@ -479,6 +485,53 @@ void TestHistUpdaterApplySplit(const xgboost::tree::TrainParam& param, float spa

}

template <typename GradientSumT>
void TestHistUpdaterExpandWithLossGuide(const xgboost::tree::TrainParam& param) {
const size_t num_rows = 3;
const size_t num_columns = 1;
const size_t n_bins = 16;

Context ctx;
ctx.UpdateAllowUnknown(Args{{"device", "sycl"}});

DeviceManager device_manager;
auto qu = device_manager.GetQueue(ctx.Device());

std::vector<float> data = {7, 3, 15};
auto p_fmat = GetDMatrixFromData(data, num_rows, num_columns);

DeviceMatrix dmat;
dmat.Init(qu, p_fmat.get());
common::GHistIndexMatrix gmat;
gmat.Init(qu, &ctx, dmat, n_bins);

std::vector<GradientPair> gpair_host = {{1, 2}, {3, 1}, {1, 1}};
USMVector<GradientPair, MemoryType::on_device> gpair(&qu, gpair_host);

RegTree tree;
FeatureInteractionConstraintHost int_constraints;
TestHistUpdater<GradientSumT> updater(&ctx, qu, param, int_constraints, p_fmat.get());
updater.SetHistSynchronizer(new BatchHistSynchronizer<GradientSumT>());
updater.SetHistRowsAdder(new BatchHistRowsAdder<GradientSumT>());
auto* row_set_collection = updater.TestInitData(gmat, gpair, *p_fmat, tree);

updater.TestExpandWithLossGuide(gmat, p_fmat.get(), &tree, gpair);

const auto& nodes = tree.GetNodes();
std::vector<float> ans(data.size());
for (size_t data_idx = 0; data_idx < data.size(); ++data_idx) {
size_t node_idx = 0;
while (!nodes[node_idx].IsLeaf()) {
node_idx = data[data_idx] < nodes[node_idx].SplitCond() ? nodes[node_idx].LeftChild() : nodes[node_idx].RightChild();
}
ans[data_idx] = nodes[node_idx].LeafValue();
}

ASSERT_NEAR(ans[0], -0.15, 1e-6);
ASSERT_NEAR(ans[1], -0.45, 1e-6);
ASSERT_NEAR(ans[2], -0.15, 1e-6);
}

TEST(SyclHistUpdater, Sampling) {
xgboost::tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"subsample", "0.7"}});
Expand Down Expand Up @@ -546,4 +599,13 @@ TEST(SyclHistUpdater, ApplySplitDence) {
TestHistUpdaterApplySplit<double>(param, 0.0, (1u << 16) + 1);
}

TEST(SyclHistUpdater, ExpandWithLossGuide) {
xgboost::tree::TrainParam param;
param.UpdateAllowUnknown(Args{{"max_depth", "2"},
{"grow_policy", "lossguide"}});

TestHistUpdaterExpandWithLossGuide<float>(param);
TestHistUpdaterExpandWithLossGuide<double>(param);
}

} // namespace xgboost::sycl::tree

0 comments on commit 3d4af4f

Please sign in to comment.