diff --git a/aten/src/ATen/SparseCsrTensorUtils.h b/aten/src/ATen/SparseCsrTensorUtils.h
index daec805bc58d8..24b5ae47df7d6 100644
--- a/aten/src/ATen/SparseCsrTensorUtils.h
+++ b/aten/src/ATen/SparseCsrTensorUtils.h
@@ -181,6 +181,38 @@ inline std::string plainIndicesName(Layout layout) {
       [&] { return "row_indices"; });
 }
 
+inline std::string compressedDimName(Layout layout) {
+  switch (layout) {
+    case kSparseCsr:
+      return "row";
+    case kSparseCsc:
+      return "column";
+    case kSparseBsr:
+      return "row block";
+    case kSparseBsc:
+      return "column block";
+    default:
+      TORCH_CHECK(false, "Not a sparse compressed layout:", layout);
+      return "";
+  }
+}
+
+inline std::string plainDimName(Layout layout) {
+  switch (layout) {
+    case kSparseCsr:
+      return "column";
+    case kSparseCsc:
+      return "row";
+    case kSparseBsr:
+      return "column block";
+    case kSparseBsc:
+      return "row block";
+    default:
+      TORCH_CHECK(false, "Not a sparse compressed layout:", layout);
+      return "";
+  }
+}
+
 inline int rowDimension(Layout layout, IntArrayRef size) {
   return size.size() - (isCompressedRow(layout) ? 2 : 1);
 }
diff --git a/aten/src/ATen/native/sparse/SparseCsrTensor.cpp b/aten/src/ATen/native/sparse/SparseCsrTensor.cpp
index 77979f55647de..1cbd1afdbda60 100644
--- a/aten/src/ATen/native/sparse/SparseCsrTensor.cpp
+++ b/aten/src/ATen/native/sparse/SparseCsrTensor.cpp
@@ -28,6 +28,7 @@
 #include <ATen/ops/_validate_sparse_csc_tensor_args_native.h>
 #include <ATen/ops/_validate_sparse_bsr_tensor_args_native.h>
 #include <ATen/ops/_validate_sparse_bsc_tensor_args_native.h>
+#include <ATen/ops/aminmax.h>
 #include <ATen/ops/ccol_indices_native.h>
 #include <ATen/ops/clone_native.h>
 #include <ATen/ops/col_indices_native.h>
@@ -58,199 +59,206 @@ namespace {
 
 } // end anonymous namespace
 
-void _validate_sparse_compressed_tensor_args_worker(const Tensor& compressed_indices, const Tensor& plain_indices, const Tensor& values, const IntArrayRef size, const Layout& layout) {
+/*
+  Validate the arguments to sparse compressed (CSR, CSC, BSR, and BSC)
+  tensor factory functions.
+
+  The CSR and BSR invariants for PyTorch are outlined in
+
+    https://pearu.github.io/csr_tensor_invariants.html
+    https://pearu.github.io/bsr_tensor_invariants.html
 
-  // Layout must be Sparse Compressed
+  that in what follows are generalized for all sparse compressed
+  formats with support to batched and dense dimensions.
+*/
+
+void _validate_sparse_compressed_tensor_args_worker(const Tensor& compressed_indices, const Tensor& plain_indices, const Tensor& values, const IntArrayRef size, const Layout& layout) {
+  // Layout must be Sparse Compressed, 2.4
   AT_DISPATCH_ALL_SPARSE_COMPRESSED_LAYOUTS(layout, "validate_sparse_compressed_tensor_args", [&]{});
 
   const std::string layout_name = layoutToString(layout, /*upper=*/ true);
   const std::string compressed_indices_name = compressedIndicesName(layout);
   const std::string plain_indices_name = plainIndicesName(layout);
+  const std::string compressed_dim_name = compressedDimName(layout);
+  const std::string plain_dim_name = plainDimName(layout);
 
   // Layout Invariants
+  // 2.1, 3.5
   TORCH_CHECK(
-      plain_indices.layout() == kStrided && plain_indices.is_contiguous(),
-      "expected ", plain_indices_name, " to be a strided and contiguous tensor");
+              plain_indices.layout() == kStrided && plain_indices.is_contiguous(),
+              "expected ", plain_indices_name, " to be a strided and contiguous tensor");
 
+  // 2.2, 3.6
   TORCH_CHECK(
-      compressed_indices.layout() == kStrided && compressed_indices.is_contiguous(),
-      "expected ", compressed_indices_name ," to be a strided and contiguous tensor");
+              compressed_indices.layout() == kStrided && compressed_indices.is_contiguous(),
+              "expected ", compressed_indices_name ," to be a strided and contiguous tensor");
 
+  // 2.3, partially 3.7
+  // TODO: allow values be contiguous along both block dimensions when the format is BSR or BSC
   TORCH_CHECK(
       values.layout() == kStrided && values.is_contiguous(),
       "expected values to be a strided and contiguous tensor");
 
+  const int base_ndim = 2;  // corresponds to compressed and plain indices
+  const int batch_ndim = compressed_indices.dim() - 1;
+  const int block_ndim = AT_DISPATCH_PLAIN_SPARSE_COMPRESSED_LAYOUTS(
+                           layout, "validate_sparse_compressed_tensor_args",
+                           [&] { return 0; }, [&] { return 2; });
+  const int dense_ndim = values.dim() - batch_ndim - block_ndim - 1;
   // Shape and Strides invariants
-  TORCH_CHECK(
-              size.size() >= 2,
-              "size of a batched ", layout_name, " tensor must have length >= 2, but got: ",
-              size.size());
-  TORCH_CHECK(
-              compressed_indices.dim() >= 1,
-              compressed_indices_name, " must have dim >= 1 but got ", compressed_indices_name, ".dim() = ",
-              compressed_indices.dim());
-  TORCH_CHECK(
-              plain_indices.dim() >= 1,
-              plain_indices_name, " must have dim >= 1 but got ", plain_indices_name, ".dim() = ",
-              plain_indices.dim());
-  TORCH_CHECK(
-              values.dim() >= 1,
-              "values must have dim >= 1 but got values.dim() = ",
-              values.dim());
 
+  // 3.2
   TORCH_CHECK(
-      compressed_indices.dim() == plain_indices.dim(),
-      "number of dimensions of ", compressed_indices_name, " and ", plain_indices_name, " must be the same but got ",
-      compressed_indices.dim(), " and ", plain_indices.dim(), ", respectively");
-
-  int block_ndim = AT_DISPATCH_PLAIN_SPARSE_COMPRESSED_LAYOUTS(
-      layout, "validate_sparse_compressed_tensor_args",
-      [&] {
-        TORCH_CHECK(
-                    compressed_indices.dim() <= values.dim(),
-                    "number of dimensions of indices (=", compressed_indices.dim(),
-                    ") must be equal or less than the number of dimensions of values (=", values.dim(), ")");
-        return 0;
-      },
-      [&] {
-        TORCH_CHECK(
-                    compressed_indices.dim() + 2 <= values.dim(),
-                    "number of dimensions of indices (=", compressed_indices.dim(),
-                    ") plus two must be equal or less than the number of dimensions of values (=", values.dim(), ")");
-        return 2;
-      });
-  int dense_ndim = values.dim() - compressed_indices.dim() - block_ndim;
-  TORCH_CHECK(dense_ndim == 0, "non-zero dense dimensions (=", dense_ndim, ") is not supported for ", layout, " layout");
+              batch_ndim >= 0,
+              compressed_indices_name, " must have dimensionality >= 1 but got ", compressed_indices.dim());
 
-  int batch_ndim = size.size() - 2 - dense_ndim;
-  TORCH_INTERNAL_ASSERT(block_ndim >= 0 && dense_ndim >=0 && batch_ndim >= 0);
+  // 3.3
+  TORCH_CHECK(
+              compressed_indices.dim() == plain_indices.dim(),
+              compressed_indices_name, " and ", plain_indices_name, " dimensionalities must be equal but got ",
+              compressed_indices.dim(), " and ", plain_indices.dim(), ", respectively");
 
+  // 3.4
+  TORCH_CHECK(
+              dense_ndim >= 0,
+              "values must have dimensionality > sum of batch and block dimensionalities (=",
+              batch_ndim, " + ", block_ndim, ") but got ", values.dim());
+  // 3.1
   TORCH_CHECK(
-      static_cast<size_t>(compressed_indices.dim()) == size.size() - 1 - dense_ndim,
-      "number of dimensions of indices must be one less than the number of dimensions of the provided size",
-      " (minus the number of dense dimensions) but got ",
-      compressed_indices.dim(), " not equal to ", size.size(), " - 1 - ", dense_ndim);
+              static_cast<int>(size.size()) == batch_ndim + base_ndim + dense_ndim,
+              "tensor dimensionality must be sum of batch, base, and dense dimensionalites (=",
+              batch_ndim, " + ", base_ndim, " + ", dense_ndim, ") but got ", size.size());
 
   // For CSR/CSC formats, we define blocksize=(1, 1) so that checking
   // the sparse compressed tensor invariants can be unified with the
   // BSR/BSC invariants.
+  // 3.10
   DimVector blocksize{
-                      (block_ndim == 2 ? std::max<int64_t>(1, values.sizes()[values.dim() - dense_ndim - 2]) : 1),
-                      (block_ndim == 2 ? std::max<int64_t>(1, values.sizes()[values.dim() - dense_ndim - 1]) : 1),
+                      (block_ndim == 2 ? std::max<int64_t>(1, values.size(batch_ndim + 1)) : 1),
+                      (block_ndim == 2 ? std::max<int64_t>(1, values.size(batch_ndim + 2)) : 1),
   };
   TORCH_INTERNAL_ASSERT(blocksize.size() == 2 && blocksize[0] > 0 && blocksize[1] > 0);
 
-  int64_t numel_per_block = blocksize[0] * blocksize[1];
-  int compressed_dim = compressedDimension(layout, size, dense_ndim);
-  int plain_dim = plainDimension(layout, size, dense_ndim);
-
-  // All batch sizes must be the same
-  DimVector batch_size = DimVector(size.slice(0, batch_ndim));
-  DimVector compressed_indices_batch_size = DimVector(compressed_indices.sizes().slice(0, compressed_indices.dim() - 1));
-  DimVector plain_indices_batch_size = DimVector(plain_indices.sizes().slice(0, plain_indices.dim() - 1));
-  DimVector values_batch_size = DimVector(values.sizes().slice(0, values.dim() - 1 - block_ndim - dense_ndim));
+  // All batch sizes must be the same and consistent with tensor batchsize, 3.1, 3.8, 3.9, 3.10
+  DimVector batchsize = DimVector(size.slice(0, batch_ndim));
+  DimVector compressed_indices_batchsize = DimVector(compressed_indices.sizes().slice(0, batch_ndim));
+  DimVector plain_indices_batchsize = DimVector(plain_indices.sizes().slice(0, batch_ndim));
+  DimVector values_batchsize = DimVector(values.sizes().slice(0, batch_ndim));
+  const int values_nnz = (values.numel() ? values.size(batch_ndim) : 0);
+  DimVector values_blocksize = DimVector(values.sizes().slice(batch_ndim + 1, block_ndim));
+  DimVector values_densesize = DimVector(values.sizes().slice(batch_ndim + 1 + block_ndim, dense_ndim));
   TORCH_CHECK(
-      batch_size == compressed_indices_batch_size &&
-      batch_size == plain_indices_batch_size &&
-      batch_size == values_batch_size,
-      "all batch dimensions of the provided size (", batch_size, "), indices (",
-      compressed_indices_batch_size,", ", plain_indices_batch_size, "), and values (",
-      values_batch_size,") must be the same.");
-
-  // A tensor constitutes of full blocks
-  for (int i=0; i<block_ndim; i++) {
-    TORCH_CHECK(size[batch_ndim + i] % blocksize[i] == 0, "tensor size(", batch_ndim + i, ") ", size[batch_ndim + i],
-                " needs to be divisible with blocksize[", i, "] ", blocksize[i]);
-  }
+      batchsize == compressed_indices_batchsize && batchsize == plain_indices_batchsize && batchsize == values_batchsize,
+      "all batch dimensions of ", compressed_indices_name," (=", compressed_indices_batchsize, "), ", plain_indices_name," (=",
+      plain_indices_batchsize, "), and values (=", values_batchsize, ") must be equal to tensor batch dimensions (=",
+      batchsize, ")");
 
-  // Note, this check also enforces `compressed_indices.size(-1) >= 1`
-  if (block_ndim == 2) {
-    TORCH_CHECK(
-                compressed_indices.size(-1) == (size[compressed_dim] / blocksize[compressed_dim - batch_ndim] + 1),
-                compressed_indices_name, ".size(-1) must be equal to size[-", (size.size() - compressed_dim),
-                "]/blocksize[",  compressed_dim - batch_ndim, "] + 1 (that is ",
-                size[compressed_dim] / blocksize[compressed_dim - batch_ndim] + 1, "), but got: ", compressed_indices.size(-1));
-    TORCH_CHECK(
-                plain_indices.numel() * numel_per_block == values.numel(),
-                "number of ", plain_indices_name, " elements must be the same as the number of blocks in values, but got ",
-                plain_indices_name, ".numel() * numel_per_block: ", plain_indices.numel() * numel_per_block,
-                ", values.numel(): ", values.numel(),", numel_per_block: ", numel_per_block);
-  } else {
-    TORCH_CHECK(
-                compressed_indices.size(-1) == (size[compressed_dim] + 1),
-                compressed_indices_name, ".size(-1) must be equal to size[-", (size.size() - compressed_dim),
-                "] + 1 (that is ",
-                size[compressed_dim] + 1, "), but got: ", compressed_indices.size(-1));
-    TORCH_CHECK(
-                plain_indices.numel() == values.numel(),
-                "number of ", plain_indices_name, " elements must be the same number of elements, but got ",
-                plain_indices_name, ".numel(): ", plain_indices.numel(),
-                ", values.numel(): ", values.numel());
+  // A tensor constitutes of full blocks, 3.1
+  for (int i=0; i<block_ndim; i++) {
+      TORCH_CHECK(size[batch_ndim + i] % blocksize[i] == 0,
+                  "tensor shape[", batch_ndim + i, "] (=", size[batch_ndim + i],
+                  ") must be divisible with blocksize[", i, "] (=", blocksize[i],
+                  ") as defined by values shape");
   }
-
+  const int nrows = size[batch_ndim] / blocksize[0];
+  const int ncols = size[batch_ndim + 1] / blocksize[1];
+  int ncompressed_dims, nplain_dims;
+  std::tie(ncompressed_dims, nplain_dims) = AT_DISPATCH_ROW_SPARSE_COMPRESSED_LAYOUTS(layout, "validate_sparse_compressed_tensor_args",
+                                                                                      [&] { return std::make_tuple(nrows, ncols); },
+                                                                                      [&] { return std::make_tuple(ncols, nrows); });
+  // 3.8
+  TORCH_CHECK(
+              compressed_indices.size(-1) == ncompressed_dims + 1,
+              compressed_indices_name, ".shape[-1] must be equal to the number of ",
+              compressed_dim_name, "s + 1 (=", ncompressed_dims + 1, "), but got ", compressed_indices.size(-1));
+  // 3.9, 3.10
+  TORCH_CHECK(
+              plain_indices.size(-1) == values_nnz,
+              plain_indices_name, ".shape[-1] must be equal to nnz (=", values_nnz,
+              ") as defined by values.shape[", batch_ndim, "], but got ", plain_indices.size(-1));
   // Type Invariants
   auto compressed_indices_type = compressed_indices.scalar_type();
   auto plain_indices_type = plain_indices.scalar_type();
+  // 1.1, 1.2, 1.3
   TORCH_CHECK(
       compressed_indices_type == plain_indices_type,
-      "both ", compressed_indices_name, " and ", plain_indices_name, " should have the same type, bot got ",
+      compressed_indices_name, " and ", plain_indices_name, " must have the same dtype, bot got ",
       compressed_indices_type, " and ", plain_indices_type, ", respectively");
   TORCH_CHECK(
       compressed_indices_type == kInt || compressed_indices_type == kLong,
-      compressed_indices_name, " and ", plain_indices_name, " must be an int32 or int64 type, but got: ",
+      compressed_indices_name, " and ", plain_indices_name, " dtype must be Int or Long, but got ",
       compressed_indices_type);
 
   // Indices invariants
   AT_DISPATCH_INDEX_TYPES(compressed_indices_type, "validate_sparse_compressed_tensor_args",
-      [&] {
+    [&] {
+      if (plain_indices.numel() > 0) {
         Tensor compressed_indices_cpu = compressed_indices.to(kCPU);
+        Tensor plain_indices_cpu = plain_indices.to(kCPU);
+        int64_t batch_compressed_stride = compressed_indices_cpu.dim() >= 2 ? compressed_indices_cpu.stride(-2) : 0;
+        int64_t batch_plain_stride = plain_indices_cpu.dim() >= 2 ? plain_indices_cpu.stride(-2) : 0;
         auto compressed_indices_data_ptr = compressed_indices_cpu.data_ptr<index_t>();
-        auto batch_stride = compressed_indices_cpu.dim() >= 2 ? compressed_indices_cpu.stride(-2) : 0;
-        auto compressed_dims = (block_ndim == 0 ? size[compressed_dim] : size[compressed_dim] / blocksize[compressed_dim - batch_ndim]);
+        auto plain_indices_data_ptr = plain_indices_cpu.data_ptr<index_t>();
         for (const auto batch_id : c10::irange(batchCount(compressed_indices_cpu))) {
-          TORCH_CHECK(
-                      compressed_indices_data_ptr[batch_id*batch_stride] == 0,
-                      "(Batch element ", batch_id, ") ",
-                      ": 0th value of ", compressed_indices_name, " must be 0, but it is ", compressed_indices_data_ptr[batch_id*batch_stride]);
-          TORCH_CHECK(
-                      compressed_indices_data_ptr[batch_id*batch_stride + compressed_indices.size(-1) - 1] == plain_indices.size(-1),
-                      "(Batch element ", batch_id, ") ",
-                      "last value of ", compressed_indices_name, " should be equal to the length of ", plain_indices_name, ".");
-          for (int i =  1; i <= compressed_dims; i++) {
-            TORCH_CHECK(
-                        compressed_indices_data_ptr[batch_id*batch_stride + i - 1] <= compressed_indices_data_ptr[batch_id*batch_stride + i],
-                        "(Batch element ", batch_id, ") ",
-                        "at position i = ", i, ", the condition ", compressed_indices_name, "[i - 1] <= ", compressed_indices_name, "[i] fails, got ",
-                        compressed_indices_data_ptr[batch_id*batch_stride + i - 1], " <= ", compressed_indices_data_ptr[batch_id*batch_stride + i]);
+          int64_t start = compressed_indices_data_ptr[batch_id*batch_compressed_stride];
+          const std::string at_batch_id = (batch_ndim > 0 ? " at batch id " + std::to_string(batch_id) : "");
+          const std::string batch_indices = (batch_ndim > 0 ? "..., " : "");
+          // 5.1
+          TORCH_CHECK(start == 0, compressed_indices_name, "[", batch_indices, "0] (=", start, ") == 0 is unsatisfied", at_batch_id);
+          for (int i = 1; i <= ncompressed_dims; i++) {
+            int64_t end = compressed_indices_data_ptr[batch_id*batch_compressed_stride + i];
+            // 5.2
+            TORCH_CHECK(end <= values_nnz,
+                        compressed_indices_name, "[", batch_indices, i, "] (=", end, ") ",
+                        "<= nnz (=", values_nnz, ") is unsatisfied", at_batch_id);
+            // 5.3
+            TORCH_CHECK(start <= end, compressed_indices_name, " must be ordered sequence but ",
+                        compressed_indices_name, "[", batch_indices, i - 1, "] (=", start, ") <= ",
+                        compressed_indices_name, "[", batch_indices, i, "] (=", end, ") is unsatisfied", at_batch_id);
+            TORCH_CHECK(end - start <= nplain_dims,
+                        compressed_indices_name, "[", batch_indices, i, "] (=", end, ") - ",
+                        compressed_indices_name, "[", batch_indices, i - 1, "] (=", start, ") <= number of ",
+                        plain_dim_name, "s (=", nplain_dims, ") is unsatisfied", at_batch_id);
+            int64_t last_plain_index = -1;
+            for (int n = start; n < end; n++) {
+              int64_t plain_index = plain_indices_data_ptr[batch_id*batch_plain_stride + n];
+              // 5.4, 5.5
+              TORCH_CHECK(0 <= plain_index && plain_index < nplain_dims,
+                          plain_indices_name, "[", batch_indices, n, "] (=", plain_index, ") is out of range (0, ", nplain_dims, ")", at_batch_id);
+              // 5.6
+              TORCH_CHECK(plain_index > last_plain_index, plain_indices_name, " must be ordered sequence of distinct integers but ",
+                          plain_indices_name, "[", batch_indices, n - 1, "] (=", last_plain_index, ") < ",
+                          plain_indices_name, "[", batch_indices, n, "] (=", plain_index, ") is unsatisfied", at_batch_id);
+              last_plain_index = plain_index;
+            }
+            start = end;
           }
         }
-        if (plain_indices.numel() > 0) {
-          TORCH_CHECK(0 <= plain_indices.min().item<index_t>(), plain_indices_name, ".min() should be greater or equal to zero");
-          TORCH_CHECK(size[plain_dim] > plain_indices.max().item<index_t>(), "size[-", (size.size() - plain_dim),"] should be greater than ", plain_indices_name, ".max()");
-        }
-      });
+      }
+    });
 
   // Device Invariants
-  TORCH_CHECK(
-      plain_indices.get_device() == compressed_indices.get_device(),
-      compressed_indices_name, " and ", plain_indices_name, " devices (",
-      compressed_indices.get_device(),
-      ", ",
-      plain_indices.get_device(),
-      ") must match");
-  TORCH_CHECK(
-      compressed_indices.get_device() == values.get_device(),
-      "device of ", compressed_indices_name, " (",
-      compressed_indices.get_device(),
-      ") must match device of values (",
-      values.get_device(),
-      ")");
+  // 4.1
   TORCH_CHECK(
       values.device().type() == kCPU || values.device().type() == kCUDA,
       "device type of values (",
       values.device().type(),
       ") must be CPU or CUDA");
-
+  // 4.2, 4.3, 4.4
+  TORCH_CHECK(
+      compressed_indices.get_device() == values.get_device(),
+      "device of ", compressed_indices_name, " (=",
+      compressed_indices.device(),
+      ") must match device of values (=",
+      values.device(),
+      ")");
+  TORCH_CHECK(
+      compressed_indices.get_device() == plain_indices.get_device(),
+      "device of ", compressed_indices_name, " (=",
+      compressed_indices.device(),
+      ") must match device of ", plain_indices_name," (=",
+      plain_indices.device(),
+      ")");
 }
 
 void _validate_sparse_compressed_tensor_args(const Tensor& compressed_indices, const Tensor& plain_indices, const Tensor& values, IntArrayRef size, Layout layout) {
@@ -360,34 +368,54 @@ DimVector _estimate_sparse_compressed_tensor_size(
     const Tensor& plain_indices,
     const Tensor& values,
     Layout layout) {
-  int block_ndim = AT_DISPATCH_PLAIN_SPARSE_COMPRESSED_LAYOUTS(layout, "estimate_sparse_compressed_tensor_size", [&] { return 0; }, [&] { return 2; });
-  int dense_ndim = values.dim() - compressed_indices.dim() - block_ndim;
+  const int block_ndim = AT_DISPATCH_PLAIN_SPARSE_COMPRESSED_LAYOUTS(layout, "estimate_sparse_compressed_tensor_size", [&] { return 0; }, [&] { return 2; });
+  const int base_ndim = 2;  // corresponds to compressed and plain indices
+  const int batch_ndim = compressed_indices.dim() - 1;
+  const std::string compressed_indices_name = compressedIndicesName(layout);
+  const std::string plain_indices_name = plainIndicesName(layout);
+  TORCH_CHECK(
+              batch_ndim >= 0,
+              compressed_indices_name, " must have dimensionality >= 1 but got ", compressed_indices.dim());
+  TORCH_CHECK(
+              compressed_indices.dim() == plain_indices.dim(),
+              compressed_indices_name, " and ", plain_indices_name, " dimensionalities must be equal but got ",
+              compressed_indices.dim(), " and ", plain_indices.dim(), ", respectively");
+  const int dense_ndim = values.dim() - batch_ndim - block_ndim - 1;
+  TORCH_CHECK(
+              dense_ndim >= 0,
+              "values must have dimensionality > sum of batch and block dimensionalities (=",
+              batch_ndim, " + ", block_ndim, ") but got ", values.dim());
   DimVector blocksize{
-                      (block_ndim == 2 ? std::max<int64_t>(1, values.sizes()[values.dim() - dense_ndim - 2]) : 1),
-                      (block_ndim == 2 ? std::max<int64_t>(1, values.sizes()[values.dim() - dense_ndim - 1]) : 1),
+                      (block_ndim == 2 ? std::max<int64_t>(1, values.size(batch_ndim + 1)) : 1),
+                      (block_ndim == 2 ? std::max<int64_t>(1, values.size(batch_ndim + 2)) : 1)
   };
-  DimVector size = DimVector(IntArrayRef(plain_indices.sizes().data(), plain_indices.dim() - 1));
-  int64_t compressed_dim = (plain_indices.size(-1) > 0 ? compressed_indices.size(-1) - 1 : 0);
-  int64_t plain_dim = AT_DISPATCH_INTEGRAL_TYPES(plain_indices.scalar_type(), "estimate_sparse_compressed_tensor_size",
-                                                 [&]() -> int64_t {
-                                                   if (plain_indices.numel() > 0) {
-                                                     return plain_indices.max().item<scalar_t>() + 1;
-                                                   } else {
-                                                     return 0;
-                                                   }
+  DimVector size = DimVector(compressed_indices.sizes().slice(0, batch_ndim));
+  int64_t ncompressed_dims = (compressed_indices.dim() > 0 && compressed_indices.size(-1) > 0 ? compressed_indices.size(-1) - 1 : 0);
+  int64_t nplain_dims = AT_DISPATCH_INTEGRAL_TYPES(plain_indices.scalar_type(), "estimate_sparse_compressed_tensor_size",
+                                                   [&]() -> int64_t {
+                                                     if (plain_indices.numel() > 0) {
+                                                       return plain_indices.max().item<scalar_t>() + 1;
+                                                     } else {
+                                                       return 0;
+                                                     }
                                                    });
   AT_DISPATCH_ROW_SPARSE_COMPRESSED_LAYOUTS(layout, "estimate_sparse_compressed_tensor_size",
       [&]{
-        size.push_back(compressed_dim * blocksize[0]);
-        size.push_back(plain_dim * blocksize[1]);
+        size.push_back(ncompressed_dims * blocksize[0]);
+        size.push_back(nplain_dims * blocksize[1]);
       },
       [&]{
-        size.push_back(plain_dim * blocksize[0]);
-        size.push_back(compressed_dim * blocksize[1]);
+        size.push_back(nplain_dims * blocksize[0]);
+        size.push_back(ncompressed_dims * blocksize[1]);
       });
   for (int i=0; i<dense_ndim; i++) {
-    size.push_back(values.size(values.dim() - dense_ndim + i));
+    int64_t j = batch_ndim + 1 + base_ndim + i;
+    size.push_back((j < values.dim() ? values.size(j) : 1));
   }
+  TORCH_CHECK(
+              static_cast<int>(size.size()) == batch_ndim + base_ndim + dense_ndim,
+              "tensor dimensionality must be sum of batch, base, and dense dimensionalites (=",
+              batch_ndim, " + ", base_ndim, " + ", dense_ndim, ") but got ", size.size());
   return size;
 }
 
diff --git a/test/expect/TestSparseCompressedCPU.test_print_SparseBSC_cpu.expect b/test/expect/TestSparseCompressedCPU.test_print_SparseBSC_cpu.expect
index 0ac8d53e73b60..305379f049f5d 100644
--- a/test/expect/TestSparseCompressedCPU.test_print_SparseBSC_cpu.expect
+++ b/test/expect/TestSparseCompressedCPU.test_print_SparseBSC_cpu.expect
@@ -1,907 +1,1355 @@
 ########## torch.float32/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4,
+                      [[ 4., 14.]]]), size=(3, 4), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]])
+        [[ 4., 14.]]])
 
-########## torch.float32/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), size=(0, 0), nnz=0,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0], dtype=torch.int32)
 # _row_indices
 tensor([], dtype=torch.int32)
 # _values
-tensor([], size=(1, 0, 0))
+tensor([], size=(0, 1, 2))
 
-########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
+                       [[8.],
+                        [9.]]]]), size=(2, 6, 2), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]])
+         [[8.],
+          [9.]]]])
 
-########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), size=(2, 3, 9, 4), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]])
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]])
 
 
 ########## torch.float64/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4, dtype=torch.float64,
+                      [[ 4., 14.]]]), size=(3, 4), nnz=4, dtype=torch.float64,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]], dtype=torch.float64)
+        [[ 4., 14.]]], dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0], dtype=torch.int32)
 # _row_indices
 tensor([], dtype=torch.int32)
 # _values
-tensor([], size=(1, 0, 0), dtype=torch.float64)
+tensor([], size=(0, 1, 2), dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
-       dtype=torch.float64, layout=torch.sparse_bsc)
+                       [[8.],
+                        [9.]]]]), size=(2, 6, 2), nnz=4, dtype=torch.float64,
+       layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]], dtype=torch.float64)
+         [[8.],
+          [9.]]]], dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), size=(2, 3, 9, 4), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]], dtype=torch.float64)
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]], dtype=torch.float64)
 
 
 ########## torch.float32/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4,
+                      [[ 4., 14.]]]), size=(3, 4), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4])
 # _row_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]])
+        [[ 4., 14.]]])
 
-########## torch.float32/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), size=(0, 0), nnz=0,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0])
 # _row_indices
 tensor([], dtype=torch.int64)
 # _values
-tensor([], size=(1, 0, 0))
+tensor([], size=(0, 1, 2))
 
-########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
+                       [[8.],
+                        [9.]]]]), size=(2, 6, 2), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]])
+         [[8.],
+          [9.]]]])
 
-########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), size=(2, 3, 9, 4), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]])
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]])
 
 
 ########## torch.float64/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4, dtype=torch.float64,
+                      [[ 4., 14.]]]), size=(3, 4), nnz=4, dtype=torch.float64,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4])
 # _row_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]], dtype=torch.float64)
+        [[ 4., 14.]]], dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0])
 # _row_indices
 tensor([], dtype=torch.int64)
 # _values
-tensor([], size=(1, 0, 0), dtype=torch.float64)
+tensor([], size=(0, 1, 2), dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
-       dtype=torch.float64, layout=torch.sparse_bsc)
+                       [[8.],
+                        [9.]]]]), size=(2, 6, 2), nnz=4, dtype=torch.float64,
+       layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]], dtype=torch.float64)
+         [[8.],
+          [9.]]]], dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), size=(2, 3, 9, 4), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]], dtype=torch.float64)
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]], dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCPU.test_print_SparseBSR_cpu.expect b/test/expect/TestSparseCompressedCPU.test_print_SparseBSR_cpu.expect
index 2de9b362e31e1..3bdf969cfab88 100644
--- a/test/expect/TestSparseCompressedCPU.test_print_SparseBSR_cpu.expect
+++ b/test/expect/TestSparseCompressedCPU.test_print_SparseBSR_cpu.expect
@@ -1,907 +1,1129 @@
-########## torch.float32/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4,
-       layout=torch.sparse_bsr)
+                      [[4.],
+                       [5.]]]), size=(4, 3), nnz=4, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]])
+        [[4.],
+         [5.]]])
 
-########## torch.float32/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), size=(0, 0), nnz=0,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0], dtype=torch.int32)
 # _col_indices
 tensor([], dtype=torch.int32)
 # _values
-tensor([], size=(1, 0, 0))
+tensor([], size=(0, 2, 1))
 
 ########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), size=(2, 2, 6), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]])
+         [[ 8., 18.]]]])
 
-########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), size=(2, 3, 4, 9), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]])
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]])
 
 
-########## torch.float64/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4, dtype=torch.float64,
+                      [[4.],
+                       [5.]]]), size=(4, 3), nnz=4, dtype=torch.float64,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]], dtype=torch.float64)
+        [[4.],
+         [5.]]], dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0], dtype=torch.int32)
 # _col_indices
 tensor([], dtype=torch.int32)
 # _values
-tensor([], size=(1, 0, 0), dtype=torch.float64)
+tensor([], size=(0, 2, 1), dtype=torch.float64)
 
 ########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), size=(2, 2, 6), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]], dtype=torch.float64)
+         [[ 8., 18.]]]], dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), size=(2, 3, 4, 9), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]], dtype=torch.float64)
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]], dtype=torch.float64)
 
 
-########## torch.float32/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4,
-       layout=torch.sparse_bsr)
+                      [[4.],
+                       [5.]]]), size=(4, 3), nnz=4, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4])
 # _col_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]])
+        [[4.],
+         [5.]]])
 
-########## torch.float32/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), size=(0, 0), nnz=0,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0])
 # _col_indices
 tensor([], dtype=torch.int64)
 # _values
-tensor([], size=(1, 0, 0))
+tensor([], size=(0, 2, 1))
 
 ########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), size=(2, 2, 6), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]])
+         [[ 8., 18.]]]])
 
-########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), size=(2, 3, 4, 9), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]])
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]])
 
 
-########## torch.float64/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), size=(2, 4), nnz=4, dtype=torch.float64,
+                      [[4.],
+                       [5.]]]), size=(4, 3), nnz=4, dtype=torch.float64,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4])
 # _col_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]], dtype=torch.float64)
+        [[4.],
+         [5.]]], dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0])
 # _col_indices
 tensor([], dtype=torch.int64)
 # _values
-tensor([], size=(1, 0, 0), dtype=torch.float64)
+tensor([], size=(0, 2, 1), dtype=torch.float64)
 
 ########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), size=(2, 2, 6), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]], dtype=torch.float64)
+         [[ 8., 18.]]]], dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), size=(2, 3, 2, 4), nnz=4,
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), size=(2, 3, 4, 9), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]], dtype=torch.float64)
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]], dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCPU.test_print_SparseCSC_cpu.expect b/test/expect/TestSparseCompressedCPU.test_print_SparseCSC_cpu.expect
index a449883a3fe20..858c3756cd1d8 100644
--- a/test/expect/TestSparseCompressedCPU.test_print_SparseCSC_cpu.expect
+++ b/test/expect/TestSparseCompressedCPU.test_print_SparseCSC_cpu.expect
@@ -1,13 +1,13 @@
 ########## torch.float32/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.])
 
@@ -27,82 +27,82 @@ tensor([])
 ########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]])
+        [5., 6., 7., 8.]])
 
 ########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]])
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]])
 
 
 ########## torch.float64/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.], dtype=torch.float64)
 
@@ -122,82 +122,82 @@ tensor([], dtype=torch.float64)
 ########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], dtype=torch.float64)
+        [5., 6., 7., 8.]], dtype=torch.float64)
 
 ########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], dtype=torch.float64)
 
 
 ########## torch.float32/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4])
 # _row_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
 tensor([1., 2., 3., 4.])
 
@@ -217,82 +217,82 @@ tensor([])
 ########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]])
+        [5., 6., 7., 8.]])
 
 ########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]])
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]])
 
 
 ########## torch.float64/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4])
 # _row_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
 tensor([1., 2., 3., 4.], dtype=torch.float64)
 
@@ -312,68 +312,68 @@ tensor([], dtype=torch.float64)
 ########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], dtype=torch.float64)
+        [5., 6., 7., 8.]], dtype=torch.float64)
 
 ########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCPU.test_print_SparseCSR_cpu.expect b/test/expect/TestSparseCompressedCPU.test_print_SparseCSR_cpu.expect
index 02476652e4b7f..267a19c4d4ecc 100644
--- a/test/expect/TestSparseCompressedCPU.test_print_SparseCSR_cpu.expect
+++ b/test/expect/TestSparseCompressedCPU.test_print_SparseCSR_cpu.expect
@@ -1,13 +1,13 @@
 ########## torch.float32/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.])
 
@@ -27,82 +27,82 @@ tensor([])
 ########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]])
+        [5., 6., 7., 8.]])
 
 ########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]])
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]])
 
 
 ########## torch.float64/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4], dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], dtype=torch.int32)
+tensor([0, 1, 0, 2], dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.], dtype=torch.float64)
 
@@ -122,82 +122,82 @@ tensor([], dtype=torch.float64)
 ########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], dtype=torch.int32)
+        [0, 3, 4]], dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], dtype=torch.int32)
+        [0, 1, 2, 0]], dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], dtype=torch.float64)
+        [5., 6., 7., 8.]], dtype=torch.float64)
 
 ########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], dtype=torch.int32)
+         [0, 3, 4]]], dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], dtype=torch.float64)
 
 
 ########## torch.float32/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4])
 # _col_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
 tensor([1., 2., 3., 4.])
 
@@ -217,82 +217,82 @@ tensor([])
 ########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]])
+        [5., 6., 7., 8.]])
 
 ########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]])
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]])
 
 
 ########## torch.float64/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), size=(2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4])
 # _col_indices
-tensor([0, 1, 0, 1])
+tensor([0, 1, 0, 2])
 # _values
 tensor([1., 2., 3., 4.], dtype=torch.float64)
 
@@ -312,68 +312,68 @@ tensor([], dtype=torch.float64)
 ########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), size=(2, 2, 2), nnz=4,
+                      [5., 6., 7., 8.]]), size=(2, 2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]])
+        [0, 3, 4]])
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]])
+        [0, 1, 2, 0]])
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], dtype=torch.float64)
+        [5., 6., 7., 8.]], dtype=torch.float64)
 
 ########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), size=(2, 3, 2, 2), nnz=4,
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), size=(2, 3, 2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]])
+         [0, 3, 4]]])
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]])
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]])
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCUDA.test_print_SparseBSC_cuda.expect b/test/expect/TestSparseCompressedCUDA.test_print_SparseBSC_cuda.expect
index d3a51cb1c939d..45f04170470b4 100644
--- a/test/expect/TestSparseCompressedCUDA.test_print_SparseBSC_cuda.expect
+++ b/test/expect/TestSparseCompressedCUDA.test_print_SparseBSC_cuda.expect
@@ -1,907 +1,1355 @@
 ########## torch.float32/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[ 4., 14.]]]), device='cuda:0', size=(3, 4), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]], device='cuda:0')
+        [[ 4., 14.]]], device='cuda:0')
 
-########## torch.float32/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), device='cuda:0', size=(0, 0), nnz=0,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0))
+tensor([], device='cuda:0', size=(0, 1, 2))
 
-########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[8.],
+                        [9.]]]]), device='cuda:0', size=(2, 6, 2), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]], device='cuda:0')
+         [[8.],
+          [9.]]]], device='cuda:0')
 
-########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), device='cuda:0', size=(2, 3, 9, 4),
        nnz=4, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]], device='cuda:0')
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]], device='cuda:0')
 
 
 ########## torch.float64/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[ 4., 14.]]]), device='cuda:0', size=(3, 4), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]], device='cuda:0', dtype=torch.float64)
+        [[ 4., 14.]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), device='cuda:0', size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0), dtype=torch.float64)
+tensor([], device='cuda:0', size=(0, 1, 2), dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[8.],
+                        [9.]]]]), device='cuda:0', size=(2, 6, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]], device='cuda:0', dtype=torch.float64)
+         [[8.],
+          [9.]]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), device='cuda:0', size=(2, 3, 9, 4),
        nnz=4, dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]], device='cuda:0', dtype=torch.float64)
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]], device='cuda:0', dtype=torch.float64)
 
 
 ########## torch.float32/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[ 4., 14.]]]), device='cuda:0', size=(3, 4), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0')
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]], device='cuda:0')
+        [[ 4., 14.]]], device='cuda:0')
 
-########## torch.float32/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), device='cuda:0', size=(0, 0), nnz=0,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0], device='cuda:0')
 # _row_indices
 tensor([], device='cuda:0', dtype=torch.int64)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0))
+tensor([], device='cuda:0', size=(0, 1, 2))
 
-########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[8.],
+                        [9.]]]]), device='cuda:0', size=(2, 6, 2), nnz=4,
        layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]], device='cuda:0')
+         [[8.],
+          [9.]]]], device='cuda:0')
 
-########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), device='cuda:0', size=(2, 3, 9, 4),
        nnz=4, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]], device='cuda:0')
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]], device='cuda:0')
 
 
 ########## torch.float64/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
+       row_indices=tensor([0, 1, 0, 2]),
        values=tensor([[[ 1., 11.]],
 
-                      [[ 2., 22.]],
+                      [[ 2., 12.]],
 
-                      [[ 3., 33.]],
+                      [[ 3., 13.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[ 4., 14.]]]), device='cuda:0', size=(3, 4), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0')
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
 tensor([[[ 1., 11.]],
 
-        [[ 2., 22.]],
+        [[ 2., 12.]],
 
-        [[ 3., 33.]],
+        [[ 3., 13.]],
 
-        [[ 4., 44.]]], device='cuda:0', dtype=torch.float64)
+        [[ 4., 14.]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0]),
        row_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 1, 2)), device='cuda:0', size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([0], device='cuda:0')
 # _row_indices
 tensor([], device='cuda:0', dtype=torch.int64)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0), dtype=torch.float64)
+tensor([], device='cuda:0', size=(0, 1, 2), dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
-       values=tensor([[[[ 1., 11.]],
+                           [0, 1, 2, 0]]),
+       values=tensor([[[[1.],
+                        [2.]],
 
-                       [[ 2., 22.]],
+                       [[2.],
+                        [3.]],
 
-                       [[ 3., 33.]],
+                       [[3.],
+                        [4.]],
 
-                       [[ 4., 44.]]],
+                       [[4.],
+                        [5.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[5.],
+                        [6.]],
 
-                       [[ 2., 22.]],
+                       [[6.],
+                        [7.]],
 
-                       [[ 3., 33.]],
+                       [[7.],
+                        [8.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[8.],
+                        [9.]]]]), device='cuda:0', size=(2, 6, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
-tensor([[[[ 1., 11.]],
+tensor([[[[1.],
+          [2.]],
 
-         [[ 2., 22.]],
+         [[2.],
+          [3.]],
 
-         [[ 3., 33.]],
+         [[3.],
+          [4.]],
 
-         [[ 4., 44.]]],
+         [[4.],
+          [5.]]],
 
 
-        [[[ 1., 11.]],
+        [[[5.],
+          [6.]],
 
-         [[ 2., 22.]],
+         [[6.],
+          [7.]],
 
-         [[ 3., 33.]],
+         [[7.],
+          [8.]],
 
-         [[ 4., 44.]]]], device='cuda:0', dtype=torch.float64)
+         [[8.],
+          [9.]]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(3, 2) ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11.],
+                         [ 2., 12.],
+                         [ 3., 13.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12.],
+                         [ 3., 13.],
+                         [ 4., 14.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13.],
+                         [ 4., 14.],
+                         [ 5., 15.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14.],
+                         [ 5., 15.],
+                         [ 6., 16.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15.],
+                         [ 6., 16.],
+                         [ 7., 17.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16.],
+                         [ 7., 17.],
+                         [ 8., 18.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17.],
+                         [ 8., 18.],
+                         [ 9., 19.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18.],
+                         [ 9., 19.],
+                         [10., 20.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19.],
+                         [10., 20.],
+                         [11., 21.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20.],
+                         [11., 21.],
+                         [12., 22.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21.],
+                         [12., 22.],
+                         [13., 23.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22.],
+                         [13., 23.],
+                         [14., 24.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23.],
+                         [14., 24.],
+                         [15., 25.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24.],
+                         [15., 25.],
+                         [16., 26.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25.],
+                         [16., 26.],
+                         [17., 27.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26.],
+                         [17., 27.],
+                         [18., 28.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27.],
+                         [18., 28.],
+                         [19., 29.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28.],
+                         [19., 29.],
+                         [20., 30.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29.],
+                         [20., 30.],
+                         [21., 31.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30.],
+                         [21., 31.],
+                         [22., 32.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31.],
+                         [22., 32.],
+                         [23., 33.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32.],
+                         [23., 33.],
+                         [24., 34.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33.],
+                         [24., 34.],
+                         [25., 35.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
+                        [[24., 34.],
+                         [25., 35.],
+                         [26., 36.]]]]]), device='cuda:0', size=(2, 3, 9, 4),
        nnz=4, dtype=torch.float64, layout=torch.sparse_bsc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11.],
+           [ 2., 12.],
+           [ 3., 13.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12.],
+           [ 3., 13.],
+           [ 4., 14.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13.],
+           [ 4., 14.],
+           [ 5., 15.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14.],
+           [ 5., 15.],
+           [ 6., 16.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15.],
+           [ 6., 16.],
+           [ 7., 17.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16.],
+           [ 7., 17.],
+           [ 8., 18.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17.],
+           [ 8., 18.],
+           [ 9., 19.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18.],
+           [ 9., 19.],
+           [10., 20.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19.],
+           [10., 20.],
+           [11., 21.]],
 
-          [[ 2., 22.]],
+          [[10., 20.],
+           [11., 21.],
+           [12., 22.]],
 
-          [[ 3., 33.]],
+          [[11., 21.],
+           [12., 22.],
+           [13., 23.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22.],
+           [13., 23.],
+           [14., 24.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23.],
+           [14., 24.],
+           [15., 25.]],
 
-          [[ 2., 22.]],
+          [[14., 24.],
+           [15., 25.],
+           [16., 26.]],
 
-          [[ 3., 33.]],
+          [[15., 25.],
+           [16., 26.],
+           [17., 27.]],
 
-          [[ 4., 44.]]],
+          [[16., 26.],
+           [17., 27.],
+           [18., 28.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27.],
+           [18., 28.],
+           [19., 29.]],
 
-          [[ 2., 22.]],
+          [[18., 28.],
+           [19., 29.],
+           [20., 30.]],
 
-          [[ 3., 33.]],
+          [[19., 29.],
+           [20., 30.],
+           [21., 31.]],
 
-          [[ 4., 44.]]],
+          [[20., 30.],
+           [21., 31.],
+           [22., 32.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31.],
+           [22., 32.],
+           [23., 33.]],
 
-          [[ 2., 22.]],
+          [[22., 32.],
+           [23., 33.],
+           [24., 34.]],
 
-          [[ 3., 33.]],
+          [[23., 33.],
+           [24., 34.],
+           [25., 35.]],
 
-          [[ 4., 44.]]]]], device='cuda:0', dtype=torch.float64)
+          [[24., 34.],
+           [25., 35.],
+           [26., 36.]]]]], device='cuda:0', dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCUDA.test_print_SparseBSR_cuda.expect b/test/expect/TestSparseCompressedCUDA.test_print_SparseBSR_cuda.expect
index 90c158c8860d4..141c0a2b1b734 100644
--- a/test/expect/TestSparseCompressedCUDA.test_print_SparseBSR_cuda.expect
+++ b/test/expect/TestSparseCompressedCUDA.test_print_SparseBSR_cuda.expect
@@ -1,907 +1,1131 @@
-########## torch.float32/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[4.],
+                       [5.]]]), device='cuda:0', size=(4, 3), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]], device='cuda:0')
+        [[4.],
+         [5.]]], device='cuda:0')
 
-########## torch.float32/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), device='cuda:0', size=(0, 0), nnz=0,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0))
+tensor([], device='cuda:0', size=(0, 2, 1))
 
 ########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), device='cuda:0', size=(2, 2, 6), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]], device='cuda:0')
+         [[ 8., 18.]]]], device='cuda:0')
 
-########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
-       nnz=4, layout=torch.sparse_bsr)
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), device='cuda:0',
+       size=(2, 3, 4, 9), nnz=4, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]], device='cuda:0')
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]], device='cuda:0')
 
 
-########## torch.float64/torch.int32/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[4.],
+                       [5.]]]), device='cuda:0', size=(4, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]], device='cuda:0', dtype=torch.float64)
+        [[4.],
+         [5.]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int32/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), device='cuda:0', size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0), dtype=torch.float64)
+tensor([], device='cuda:0', size=(0, 2, 1), dtype=torch.float64)
 
 ########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), device='cuda:0', size=(2, 2, 6), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]], device='cuda:0', dtype=torch.float64)
+         [[ 8., 18.]]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
-       nnz=4, dtype=torch.float64, layout=torch.sparse_bsr)
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), device='cuda:0',
+       size=(2, 3, 4, 9), nnz=4, dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]], device='cuda:0', dtype=torch.float64)
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]], device='cuda:0', dtype=torch.float64)
 
 
-########## torch.float32/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[4.],
+                       [5.]]]), device='cuda:0', size=(4, 3), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0')
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]], device='cuda:0')
+        [[4.],
+         [5.]]], device='cuda:0')
 
-########## torch.float32/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float32/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), device='cuda:0', size=(0, 0), nnz=0,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0], device='cuda:0')
 # _col_indices
 tensor([], device='cuda:0', dtype=torch.int64)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0))
+tensor([], device='cuda:0', size=(0, 2, 1))
 
 ########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), device='cuda:0', size=(2, 2, 6), nnz=4,
        layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]], device='cuda:0')
+         [[ 8., 18.]]]], device='cuda:0')
 
-########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
-       nnz=4, layout=torch.sparse_bsr)
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), device='cuda:0',
+       size=(2, 3, 4, 9), nnz=4, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]], device='cuda:0')
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]], device='cuda:0')
 
 
-########## torch.float64/torch.int64/batch_shape=()/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([[[ 1., 11.]],
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([[[1.],
+                       [2.]],
 
-                      [[ 2., 22.]],
+                      [[2.],
+                       [3.]],
 
-                      [[ 3., 33.]],
+                      [[3.],
+                       [4.]],
 
-                      [[ 4., 44.]]]), device='cuda:0', size=(2, 4), nnz=4,
+                      [[4.],
+                       [5.]]]), device='cuda:0', size=(4, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0')
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
-tensor([[[ 1., 11.]],
+tensor([[[1.],
+         [2.]],
 
-        [[ 2., 22.]],
+        [[2.],
+         [3.]],
 
-        [[ 3., 33.]],
+        [[3.],
+         [4.]],
 
-        [[ 4., 44.]]], device='cuda:0', dtype=torch.float64)
+        [[4.],
+         [5.]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=()/block_shape=(0, 0) ##########
+########## torch.float64/torch.int64/batch_shape=()/block_shape=(2, 1) ##########
 # sparse tensor
 tensor(crow_indices=tensor([0]),
        col_indices=tensor([], size=(0,)),
-       values=tensor([], size=(1, 0, 0)), device='cuda:0', size=(0, 0), nnz=0,
+       values=tensor([], size=(0, 2, 1)), device='cuda:0', size=(0, 0), nnz=0,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([0], device='cuda:0')
 # _col_indices
 tensor([], device='cuda:0', dtype=torch.int64)
 # _values
-tensor([], device='cuda:0', size=(1, 0, 0), dtype=torch.float64)
+tensor([], device='cuda:0', size=(0, 2, 1), dtype=torch.float64)
 
 ########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=(1, 2) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[[[ 1., 11.]],
 
-                       [[ 2., 22.]],
+                       [[ 2., 12.]],
 
-                       [[ 3., 33.]],
+                       [[ 3., 13.]],
 
-                       [[ 4., 44.]]],
+                       [[ 4., 14.]]],
 
 
-                      [[[ 1., 11.]],
+                      [[[ 5., 15.]],
 
-                       [[ 2., 22.]],
+                       [[ 6., 16.]],
 
-                       [[ 3., 33.]],
+                       [[ 7., 17.]],
 
-                       [[ 4., 44.]]]]), device='cuda:0', size=(2, 2, 4), nnz=4,
+                       [[ 8., 18.]]]]), device='cuda:0', size=(2, 2, 6), nnz=4,
        dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
 tensor([[[[ 1., 11.]],
 
-         [[ 2., 22.]],
+         [[ 2., 12.]],
 
-         [[ 3., 33.]],
+         [[ 3., 13.]],
 
-         [[ 4., 44.]]],
+         [[ 4., 14.]]],
 
 
-        [[[ 1., 11.]],
+        [[[ 5., 15.]],
 
-         [[ 2., 22.]],
+         [[ 6., 16.]],
 
-         [[ 3., 33.]],
+         [[ 7., 17.]],
 
-         [[ 4., 44.]]]], device='cuda:0', dtype=torch.float64)
+         [[ 8., 18.]]]], device='cuda:0', dtype=torch.float64)
 
-########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(1, 2) ##########
+########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=(2, 3) ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
 
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[[[ 1., 11.]],
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[[[ 1., 11., 21.],
+                         [ 2., 12., 22.]],
 
-                        [[ 2., 22.]],
+                        [[ 2., 12., 22.],
+                         [ 3., 13., 23.]],
 
-                        [[ 3., 33.]],
+                        [[ 3., 13., 23.],
+                         [ 4., 14., 24.]],
 
-                        [[ 4., 44.]]],
+                        [[ 4., 14., 24.],
+                         [ 5., 15., 25.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 5., 15., 25.],
+                         [ 6., 16., 26.]],
 
-                        [[ 2., 22.]],
+                        [[ 6., 16., 26.],
+                         [ 7., 17., 27.]],
 
-                        [[ 3., 33.]],
+                        [[ 7., 17., 27.],
+                         [ 8., 18., 28.]],
 
-                        [[ 4., 44.]]],
+                        [[ 8., 18., 28.],
+                         [ 9., 19., 29.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[ 9., 19., 29.],
+                         [10., 20., 30.]],
 
-                        [[ 2., 22.]],
+                        [[10., 20., 30.],
+                         [11., 21., 31.]],
 
-                        [[ 3., 33.]],
+                        [[11., 21., 31.],
+                         [12., 22., 32.]],
 
-                        [[ 4., 44.]]]],
+                        [[12., 22., 32.],
+                         [13., 23., 33.]]]],
 
 
 
-                      [[[[ 1., 11.]],
+                      [[[[13., 23., 33.],
+                         [14., 24., 34.]],
 
-                        [[ 2., 22.]],
+                        [[14., 24., 34.],
+                         [15., 25., 35.]],
 
-                        [[ 3., 33.]],
+                        [[15., 25., 35.],
+                         [16., 26., 36.]],
 
-                        [[ 4., 44.]]],
+                        [[16., 26., 36.],
+                         [17., 27., 37.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[17., 27., 37.],
+                         [18., 28., 38.]],
 
-                        [[ 2., 22.]],
+                        [[18., 28., 38.],
+                         [19., 29., 39.]],
 
-                        [[ 3., 33.]],
+                        [[19., 29., 39.],
+                         [20., 30., 40.]],
 
-                        [[ 4., 44.]]],
+                        [[20., 30., 40.],
+                         [21., 31., 41.]]],
 
 
-                       [[[ 1., 11.]],
+                       [[[21., 31., 41.],
+                         [22., 32., 42.]],
 
-                        [[ 2., 22.]],
+                        [[22., 32., 42.],
+                         [23., 33., 43.]],
 
-                        [[ 3., 33.]],
+                        [[23., 33., 43.],
+                         [24., 34., 44.]],
 
-                        [[ 4., 44.]]]]]), device='cuda:0', size=(2, 3, 2, 4),
-       nnz=4, dtype=torch.float64, layout=torch.sparse_bsr)
+                        [[24., 34., 44.],
+                         [25., 35., 45.]]]]]), device='cuda:0',
+       size=(2, 3, 4, 9), nnz=4, dtype=torch.float64, layout=torch.sparse_bsr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[[[ 1., 11.]],
+tensor([[[[[ 1., 11., 21.],
+           [ 2., 12., 22.]],
 
-          [[ 2., 22.]],
+          [[ 2., 12., 22.],
+           [ 3., 13., 23.]],
 
-          [[ 3., 33.]],
+          [[ 3., 13., 23.],
+           [ 4., 14., 24.]],
 
-          [[ 4., 44.]]],
+          [[ 4., 14., 24.],
+           [ 5., 15., 25.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 5., 15., 25.],
+           [ 6., 16., 26.]],
 
-          [[ 2., 22.]],
+          [[ 6., 16., 26.],
+           [ 7., 17., 27.]],
 
-          [[ 3., 33.]],
+          [[ 7., 17., 27.],
+           [ 8., 18., 28.]],
 
-          [[ 4., 44.]]],
+          [[ 8., 18., 28.],
+           [ 9., 19., 29.]]],
 
 
-         [[[ 1., 11.]],
+         [[[ 9., 19., 29.],
+           [10., 20., 30.]],
 
-          [[ 2., 22.]],
+          [[10., 20., 30.],
+           [11., 21., 31.]],
 
-          [[ 3., 33.]],
+          [[11., 21., 31.],
+           [12., 22., 32.]],
 
-          [[ 4., 44.]]]],
+          [[12., 22., 32.],
+           [13., 23., 33.]]]],
 
 
 
-        [[[[ 1., 11.]],
+        [[[[13., 23., 33.],
+           [14., 24., 34.]],
 
-          [[ 2., 22.]],
+          [[14., 24., 34.],
+           [15., 25., 35.]],
 
-          [[ 3., 33.]],
+          [[15., 25., 35.],
+           [16., 26., 36.]],
 
-          [[ 4., 44.]]],
+          [[16., 26., 36.],
+           [17., 27., 37.]]],
 
 
-         [[[ 1., 11.]],
+         [[[17., 27., 37.],
+           [18., 28., 38.]],
 
-          [[ 2., 22.]],
+          [[18., 28., 38.],
+           [19., 29., 39.]],
 
-          [[ 3., 33.]],
+          [[19., 29., 39.],
+           [20., 30., 40.]],
 
-          [[ 4., 44.]]],
+          [[20., 30., 40.],
+           [21., 31., 41.]]],
 
 
-         [[[ 1., 11.]],
+         [[[21., 31., 41.],
+           [22., 32., 42.]],
 
-          [[ 2., 22.]],
+          [[22., 32., 42.],
+           [23., 33., 43.]],
 
-          [[ 3., 33.]],
+          [[23., 33., 43.],
+           [24., 34., 44.]],
 
-          [[ 4., 44.]]]]], device='cuda:0', dtype=torch.float64)
+          [[24., 34., 44.],
+           [25., 35., 45.]]]]], device='cuda:0', dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCUDA.test_print_SparseCSC_cuda.expect b/test/expect/TestSparseCompressedCUDA.test_print_SparseCSC_cuda.expect
index 4292bfcd21994..1cf98608b6091 100644
--- a/test/expect/TestSparseCompressedCUDA.test_print_SparseCSC_cuda.expect
+++ b/test/expect/TestSparseCompressedCUDA.test_print_SparseCSC_cuda.expect
@@ -1,13 +1,13 @@
 ########## torch.float32/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0')
 
@@ -27,82 +27,82 @@ tensor([], device='cuda:0')
 ########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 3, 2),
        nnz=4, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0')
+        [5., 6., 7., 8.]], device='cuda:0')
 
 ########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, layout=torch.sparse_csc)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 3, 2), nnz=4, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0')
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0')
 
 
 ########## torch.float64/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0', dtype=torch.float64)
 
@@ -122,82 +122,82 @@ tensor([], device='cuda:0', dtype=torch.float64)
 ########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 3, 2),
        nnz=4, dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0', dtype=torch.float64)
+        [5., 6., 7., 8.]], device='cuda:0', dtype=torch.float64)
 
 ########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, dtype=torch.float64, layout=torch.sparse_csc)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 3, 2), nnz=4, dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0', dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0', dtype=torch.float64)
 
 
 ########## torch.float32/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(3, 2), nnz=4,
        layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0')
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0')
 
@@ -217,82 +217,82 @@ tensor([], device='cuda:0')
 ########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 3, 2),
        nnz=4, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0')
+        [5., 6., 7., 8.]], device='cuda:0')
 
 ########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, layout=torch.sparse_csc)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 3, 2), nnz=4, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0')
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0')
 
 
 ########## torch.float64/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([0, 2, 4]),
-       row_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       row_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(3, 2), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([0, 2, 4], device='cuda:0')
 # _row_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0', dtype=torch.float64)
 
@@ -312,68 +312,68 @@ tensor([], device='cuda:0', dtype=torch.float64)
 ########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        row_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 3, 2),
        nnz=4, dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _row_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0', dtype=torch.float64)
+        [5., 6., 7., 8.]], device='cuda:0', dtype=torch.float64)
 
 ########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(ccol_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        row_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, dtype=torch.float64, layout=torch.sparse_csc)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 3, 2), nnz=4, dtype=torch.float64, layout=torch.sparse_csc)
 # _ccol_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _row_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0', dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0', dtype=torch.float64)
 
diff --git a/test/expect/TestSparseCompressedCUDA.test_print_SparseCSR_cuda.expect b/test/expect/TestSparseCompressedCUDA.test_print_SparseCSR_cuda.expect
index 918f2570807f7..e23f254bccf9d 100644
--- a/test/expect/TestSparseCompressedCUDA.test_print_SparseCSR_cuda.expect
+++ b/test/expect/TestSparseCompressedCUDA.test_print_SparseCSR_cuda.expect
@@ -1,13 +1,13 @@
 ########## torch.float32/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0')
 
@@ -27,82 +27,82 @@ tensor([], device='cuda:0')
 ########## torch.float32/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 2, 3),
        nnz=4, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0')
+        [5., 6., 7., 8.]], device='cuda:0')
 
 ########## torch.float32/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, layout=torch.sparse_csr)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 2, 3), nnz=4, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0')
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0')
 
 
 ########## torch.float64/torch.int32/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0', dtype=torch.int32)
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0', dtype=torch.int32)
+tensor([0, 1, 0, 2], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0', dtype=torch.float64)
 
@@ -122,82 +122,82 @@ tensor([], device='cuda:0', dtype=torch.float64)
 ########## torch.float64/torch.int32/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 2, 3),
        nnz=4, dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0', dtype=torch.int32)
+        [0, 3, 4]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0', dtype=torch.int32)
+        [0, 1, 2, 0]], device='cuda:0', dtype=torch.int32)
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0', dtype=torch.float64)
+        [5., 6., 7., 8.]], device='cuda:0', dtype=torch.float64)
 
 ########## torch.float64/torch.int32/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, dtype=torch.float64, layout=torch.sparse_csr)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 2, 3), nnz=4, dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0', dtype=torch.int32)
+         [0, 3, 4]]], device='cuda:0', dtype=torch.int32)
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0', dtype=torch.int32)
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0', dtype=torch.int32)
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0', dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0', dtype=torch.float64)
 
 
 ########## torch.float32/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 3), nnz=4,
        layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0')
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0')
 
@@ -217,82 +217,82 @@ tensor([], device='cuda:0')
 ########## torch.float32/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 2, 3),
        nnz=4, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0')
+        [5., 6., 7., 8.]], device='cuda:0')
 
 ########## torch.float32/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, layout=torch.sparse_csr)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 2, 3), nnz=4, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0')
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0')
 
 
 ########## torch.float64/torch.int64/batch_shape=()/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([0, 2, 4]),
-       col_indices=tensor([0, 1, 0, 1]),
-       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 2), nnz=4,
+       col_indices=tensor([0, 1, 0, 2]),
+       values=tensor([1., 2., 3., 4.]), device='cuda:0', size=(2, 3), nnz=4,
        dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([0, 2, 4], device='cuda:0')
 # _col_indices
-tensor([0, 1, 0, 1], device='cuda:0')
+tensor([0, 1, 0, 2], device='cuda:0')
 # _values
 tensor([1., 2., 3., 4.], device='cuda:0', dtype=torch.float64)
 
@@ -312,68 +312,68 @@ tensor([], device='cuda:0', dtype=torch.float64)
 ########## torch.float64/torch.int64/batch_shape=(2,)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[0, 2, 4],
-                            [0, 2, 4]]),
+                            [0, 3, 4]]),
        col_indices=tensor([[0, 1, 0, 1],
-                           [0, 1, 0, 1]]),
+                           [0, 1, 2, 0]]),
        values=tensor([[1., 2., 3., 4.],
-                      [1., 2., 3., 4.]]), device='cuda:0', size=(2, 2, 2),
+                      [5., 6., 7., 8.]]), device='cuda:0', size=(2, 2, 3),
        nnz=4, dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[0, 2, 4],
-        [0, 2, 4]], device='cuda:0')
+        [0, 3, 4]], device='cuda:0')
 # _col_indices
 tensor([[0, 1, 0, 1],
-        [0, 1, 0, 1]], device='cuda:0')
+        [0, 1, 2, 0]], device='cuda:0')
 # _values
 tensor([[1., 2., 3., 4.],
-        [1., 2., 3., 4.]], device='cuda:0', dtype=torch.float64)
+        [5., 6., 7., 8.]], device='cuda:0', dtype=torch.float64)
 
 ########## torch.float64/torch.int64/batch_shape=(2, 3)/block_shape=() ##########
 # sparse tensor
 tensor(crow_indices=tensor([[[0, 2, 4],
-                             [0, 2, 4],
-                             [0, 2, 4]],
+                             [0, 3, 4],
+                             [0, 1, 4]],
 
-                            [[0, 2, 4],
+                            [[0, 1, 4],
                              [0, 2, 4],
-                             [0, 2, 4]]]),
+                             [0, 3, 4]]]),
        col_indices=tensor([[[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]],
-
-                           [[0, 1, 0, 1],
-                            [0, 1, 0, 1],
-                            [0, 1, 0, 1]]]),
-       values=tensor([[[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]],
-
-                      [[1., 2., 3., 4.],
-                       [1., 2., 3., 4.],
-                       [1., 2., 3., 4.]]]), device='cuda:0', size=(2, 3, 2, 2),
-       nnz=4, dtype=torch.float64, layout=torch.sparse_csr)
+                            [0, 1, 2, 0],
+                            [0, 0, 1, 2]],
+
+                           [[1, 0, 1, 2],
+                            [0, 2, 0, 1],
+                            [0, 1, 2, 1]]]),
+       values=tensor([[[ 1.,  2.,  3.,  4.],
+                       [ 5.,  6.,  7.,  8.],
+                       [ 9., 10., 11., 12.]],
+
+                      [[13., 14., 15., 16.],
+                       [17., 18., 19., 20.],
+                       [21., 22., 23., 24.]]]), device='cuda:0',
+       size=(2, 3, 2, 3), nnz=4, dtype=torch.float64, layout=torch.sparse_csr)
 # _crow_indices
 tensor([[[0, 2, 4],
-         [0, 2, 4],
-         [0, 2, 4]],
+         [0, 3, 4],
+         [0, 1, 4]],
 
-        [[0, 2, 4],
+        [[0, 1, 4],
          [0, 2, 4],
-         [0, 2, 4]]], device='cuda:0')
+         [0, 3, 4]]], device='cuda:0')
 # _col_indices
 tensor([[[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]],
+         [0, 1, 2, 0],
+         [0, 0, 1, 2]],
 
-        [[0, 1, 0, 1],
-         [0, 1, 0, 1],
-         [0, 1, 0, 1]]], device='cuda:0')
+        [[1, 0, 1, 2],
+         [0, 2, 0, 1],
+         [0, 1, 2, 1]]], device='cuda:0')
 # _values
-tensor([[[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]],
+tensor([[[ 1.,  2.,  3.,  4.],
+         [ 5.,  6.,  7.,  8.],
+         [ 9., 10., 11., 12.]],
 
-        [[1., 2., 3., 4.],
-         [1., 2., 3., 4.],
-         [1., 2., 3., 4.]]], device='cuda:0', dtype=torch.float64)
+        [[13., 14., 15., 16.],
+         [17., 18., 19., 20.],
+         [21., 22., 23., 24.]]], device='cuda:0', dtype=torch.float64)
 
diff --git a/test/test_sparse_csr.py b/test/test_sparse_csr.py
index ef15c5be8e3e5..b02ee5c0e7f9d 100644
--- a/test/test_sparse_csr.py
+++ b/test/test_sparse_csr.py
@@ -14,7 +14,7 @@
      precisionOverride, skipMeta, skipCUDAIf, skipCUDAIfRocm, skipCPUIfNoMklSparse)
 from torch.testing._internal.common_methods_invocations import \
     (op_db, sparse_csr_unary_ufuncs, ReductionOpInfo)
-from torch.testing._internal.common_cuda import _get_torch_cuda_version, CUDA11OrLater
+from torch.testing._internal.common_cuda import _get_torch_cuda_version, CUDA11OrLater, TEST_CUDA
 from torch.testing._internal.common_dtype import (
     floating_types, all_types_and_complex_and, floating_and_complex_types, floating_types_and,
     all_types_and_complex, floating_and_complex_types_and
@@ -171,46 +171,137 @@ def genTensor(self, size, nnz, *, layout, device=None, dtype=torch.float, index_
             device = self.device_type
         return self.genSparseCompressedTensor(size, nnz, device=device, dtype=dtype, index_dtype=index_dtype, layout=layout)
 
-    def _generate_small_inputs(self, layout, device, dtype, index_dtype):
+    def _generate_small_inputs_utils(self, layout, device=None, dtype=None):
+
+        def shape(shape, basedim=0, blocksize=(1, 1)):
+            # Below, we define compressed and plain indices that
+            # correspond to row compressed tensors. In order to reuse
+            # the indices tensors for column compressed tensors, we
+            # swap the row and columns in shape dims (basedim and
+            # basedim + 1, respectively) to obtain the correct shape
+            # for column compressed tensors. Batch and dense
+            # dimensions remain as they are.
+            #
+            # Similarly, we reuse indices of non-block tensors for
+            # block tensors, that means, we'll need to multiply the
+            # base shape of the non-block tensor with blocksize to get
+            # the base shape of a block tensor.
+            if layout is torch.sparse_csc:
+                shape = shape[:basedim] + (shape[basedim + 1], shape[basedim]) + shape[basedim + 2:]
+            elif layout is torch.sparse_bsc:
+                shape = shape[:basedim] + (shape[basedim + 1] * blocksize[1], shape[basedim] * blocksize[0]) + shape[basedim + 2:]
+            elif layout is torch.sparse_bsr:
+                shape = shape[:basedim] + (shape[basedim] * blocksize[0], shape[basedim + 1] * blocksize[1]) + shape[basedim + 2:]
+            return shape
+
+        def values(lst, basedim=0, blocksize=(1, 1), device=device, dtype=dtype):
+            # Below, we define values for non-blocked tensors. To
+            # reuse these for blocked tensors, we replace all values
+            # in lst with a double-list that "shape" corresponds to
+            # blocksize.
+            if layout in {torch.sparse_bsr, torch.sparse_bsc}:
+
+                if layout is torch.sparse_bsc:
+                    blocksize = tuple(reversed(blocksize))
+
+                if not lst:
+                    return torch.tensor(lst, device=device, dtype=dtype).reshape(0, *blocksize)
+
+                def list_add(lst, value):
+                    if isinstance(lst, list):
+                        return [list_add(item, value) for item in lst]
+                    return lst + value
+
+                def apply_block(value, bdim):
+                    if isinstance(value, list) and bdim >= 0:
+                        return [apply_block(item, bdim - 1) for item in value]
+                    new_value = []
+                    for i in range(blocksize[0]):
+                        row = []
+                        for j in range(blocksize[1]):
+                            row.append(list_add(value, i + 10 * j))
+                        new_value.append(row)
+                    return new_value
+
+                lst = apply_block(lst, basedim)
+
+            return torch.tensor(lst, device=device, dtype=dtype)
+
+        return shape, values
+
+    def _generate_small_inputs(self, layout, device, dtype, index_dtype,
+                               enable_batched=True,
+                               enable_hybrid=False):
         """Generator of inputs to sparse compressed tensor factory functions.
 
         The input is defined as a 4-tuple:
           compressed_indices, plain_indices, values, expected_size_from_shape_inference
         """
-        from operator import mul
-        from functools import reduce
-        if layout in {torch.sparse_csr, torch.sparse_csc}:
+
+        # TODO: set enable_hybrid default to True when Sparse
+        # Compressed tensors support dense dimensions
+
+        shape, values = self._generate_small_inputs_utils(layout, device, dtype)
+
+        # a regular tensor
+        yield (torch.tensor([0, 2, 4], device=device, dtype=index_dtype),
+               torch.tensor([0, 1, 0, 2], device=device, dtype=index_dtype),
+               values([1, 2, 3, 4], 0, (2, 1)),
+               shape((2, 3), 0, (2, 1)))
+
+        # a tensor with zero dimensions
+        yield (torch.tensor([0, ], device=device, dtype=index_dtype),
+               torch.tensor([], device=device, dtype=index_dtype),
+               values([], 0, (2, 1)),
+               shape((0, 0), 0, (2, 1)))
+
+        if enable_batched:
+            # a batched tensor with one batch dimension
+            yield (torch.tensor([[0, 2, 4], [0, 3, 4]], device=device, dtype=index_dtype),
+                   torch.tensor([[0, 1, 0, 1], [0, 1, 2, 0]], device=device, dtype=index_dtype),
+                   values([[1, 2, 3, 4], [5, 6, 7, 8]], 1, (1, 2)),
+                   shape((2, 2, 3), 1, (1, 2)))
+
+            # a batched tensor with two batch dimensions
+            yield (torch.tensor([[[0, 2, 4], [0, 3, 4], [0, 1, 4]],
+                                 [[0, 1, 4], [0, 2, 4], [0, 3, 4]]],
+                                device=device, dtype=index_dtype),
+                   torch.tensor([[[0, 1, 0, 1], [0, 1, 2, 0], [0, 0, 1, 2]],
+                                 [[1, 0, 1, 2], [0, 2, 0, 1], [0, 1, 2, 1]]],
+                                device=device, dtype=index_dtype),
+                   values([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]],
+                           [[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]]], 2, (2, 3)),
+                   shape((2, 3, 2, 3), 2, (2, 3)))
+
+        if enable_hybrid:
+            # a tensor with one dense dimension
             yield (torch.tensor([0, 2, 4], device=device, dtype=index_dtype),
-                   torch.tensor([0, 1, 0, 1], device=device, dtype=index_dtype),
-                   torch.tensor([1, 2, 3, 4], device=device, dtype=dtype),
-                   (2, 2))
-            yield (torch.tensor([0, ], device=device, dtype=index_dtype),
-                   torch.tensor([], device=device, dtype=index_dtype),
-                   torch.tensor([], device=device, dtype=dtype),
-                   (0, 0))
-            for batch_shape in [(2,), (2, 3)]:
-                prod = reduce(mul, batch_shape, 1)
-                yield (torch.tensor([0, 2, 4], device=device, dtype=index_dtype).repeat(prod, 1).reshape(*batch_shape, -1),
-                       torch.tensor([0, 1, 0, 1], device=device, dtype=index_dtype).repeat(prod, 1).reshape(*batch_shape, -1),
-                       torch.tensor([1, 2, 3, 4], device=device, dtype=dtype).repeat(prod, 1).reshape(*batch_shape, -1),
-                       (*batch_shape, 2, 2))
-        else:
-            assert layout in {torch.sparse_bsr, torch.sparse_bsc}
+                   torch.tensor([0, 1, 0, 2], device=device, dtype=index_dtype),
+                   values([[1, 11], [2, 12], [3, 13], [4, 14]], 0, (3, 2)),
+                   shape((2, 3, 2), 0, (3, 2)))
+
+            # a tensor with two dense dimensions
             yield (torch.tensor([0, 2, 4], device=device, dtype=index_dtype),
-                   torch.tensor([0, 1, 0, 1], device=device, dtype=index_dtype),
-                   torch.tensor([[[1, 11]], [[2, 22]], [[3, 33]], [[4, 44]]], device=device, dtype=dtype),
-                   (2, 4))
-            yield (torch.tensor([0, ], device=device, dtype=index_dtype),
-                   torch.tensor([], device=device, dtype=index_dtype),
-                   torch.tensor([], device=device, dtype=dtype).reshape(1, 0, 0),
-                   (0, 0))
-            for batch_shape in [(2,), (2, 3)]:
-                prod = reduce(mul, batch_shape, 1)
-                yield (torch.tensor([0, 2, 4], device=device, dtype=index_dtype).repeat(prod, 1).reshape(*batch_shape, -1),
-                       torch.tensor([0, 1, 0, 1], device=device, dtype=index_dtype).repeat(prod, 1).reshape(*batch_shape, -1),
-                       torch.tensor([[[1, 11]], [[2, 22]], [[3, 33]], [[4, 44]]],
-                                    device=device, dtype=dtype).repeat(prod, 1, 1).reshape(*batch_shape, 4, 1, 2),
-                       (*batch_shape, 2, 4))
+                   torch.tensor([0, 1, 0, 2], device=device, dtype=index_dtype),
+                   values([[[1, 11]], [[2, 12]], [[3, 13]], [[4, 14]]], 0, (2, 3)),
+                   shape((2, 3, 4, 2), 0, (2, 3)))
+
+        if enable_batched and enable_hybrid:
+            # a batched tensor with two batch dimensions and two dense dimensions
+            yield (torch.tensor([[[0, 2, 4], [0, 3, 4], [0, 1, 4]],
+                                 [[0, 1, 4], [0, 2, 4], [0, 3, 4]]],
+                                device=device, dtype=index_dtype),
+                   torch.tensor([[[0, 1, 0, 1], [0, 1, 2, 0], [0, 0, 1, 2]],
+                                 [[1, 0, 1, 2], [0, 2, 0, 1], [0, 1, 2, 1]]],
+                                device=device, dtype=index_dtype),
+                   values([[[[[1], [11]], [[2], [12]], [[3], [13]], [[4], [14]]],
+                            [[[5], [15]], [[6], [16]], [[7], [17]], [[8], [18]]],
+                            [[[9], [19]], [[10], [20]], [[11], [21]], [[12], [22]]]],
+                           [[[[3], [13]], [[4], [14]], [[5], [15]], [[6], [16]]],
+                            [[[7], [17]], [[8], [18]], [[9], [19]], [[10], [20]]],
+                            [[[11], [21]], [[12], [22]], [[13], [23]], [[14], [24]]]]],
+                          2, (3, 2)),
+                   shape((2, 3, 2, 3, 2, 1), 2, (3, 2)))
 
     @all_sparse_compressed_layouts()
     @onlyCPU
@@ -331,19 +422,6 @@ def test_print(self, layout, device):
                     batch_shape = tuple(size[:-2])
                     block_shape = tuple(values.shape[-2:]) if layout in {torch.sparse_bsr, torch.sparse_bsc} else ()
                     blocksize0, blocksize1 = block_shape if layout in {torch.sparse_bsr, torch.sparse_bsc} else (1, 1)
-                    if size not in [(2 * blocksize0, 2 * blocksize1), (0, 0),
-                                    (2, 3, 2 * blocksize0, 2 * blocksize1), (2, 2 * blocksize0, 2 * blocksize1)]:
-                        # Skip inputs that are not in the list of
-                        # expected sizes to ensure the stability of
-                        # test_print in the case
-                        # _generate_small_inputs is extended with new
-                        # inputs
-                        continue
-                    if block_shape not in [(), (0, 0), (1, 2)]:
-                        # Skip inputs that are not in the list of
-                        # expected block sizes to ensure test_print
-                        # stability.
-                        continue
                     printed.append("########## {}/{}/batch_shape={}/block_shape={} ##########".format(
                         dtype, index_dtype, batch_shape, block_shape))
                     x = torch.sparse_compressed_tensor(compressed_indices,
@@ -502,6 +580,222 @@ def test_consistency(self, layout, device, dtype, op):
             raise ValueError("Expected at least one sample with keepdim and/or explicit mask for reductions.")
 
 
+    @skipMeta
+    @all_sparse_compressed_layouts()
+    @dtypes(*all_types_and_complex_and(torch.half, torch.bool, torch.bfloat16))
+    def test_validate(self, layout, device, dtype):
+        for index_dtype in [torch.int32, torch.int64]:
+            for compressed_indices, plain_indices, values, size in self._generate_small_inputs(
+                    layout, device, dtype, index_dtype, enable_batched=True, enable_hybrid=True):
+                torch._validate_sparse_compressed_tensor_args(compressed_indices, plain_indices, values, size, layout)
+
+    def _generate_invalid_input(self, layout):
+
+        shape, values = self._generate_small_inputs_utils(layout)
+
+        yield ('incontiguous compressed_indices',
+               torch.tensor([0, -1, 2, -1, 4, -1])[::2],
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               'expected compressed_indices to be a strided and contiguous tensor')
+
+        yield ('incontiguous plain_indices',
+               torch.tensor([0, 2, 4]),
+               torch.tensor([0, -1, 1, -1, 0, -1, 2, -1])[::2],
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               'expected plain_indices to be a strided and contiguous tensor')
+
+        yield ('incontiguous values',
+               torch.tensor([0, 2, 4]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 1, 2, 2, 3, 3, 4, 4])[::2],
+               shape((2, 3)),
+               'expected values to be a strided and contiguous tensor')
+
+        yield ('0-D compressed_indices',
+               torch.tensor(0),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               'compressed_indices must have dimensionality >= 1 but got 0')
+
+        yield ('compressed/plain_indices mismatch of dimensionalites',
+               torch.tensor([[0, 2, 4]]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               'compressed_indices and plain_indices dimensionalities must be equal but got 2 and 1, respectively')
+
+        if layout in {torch.sparse_csr, torch.sparse_csc}:
+            yield ('indices and values mismatch of dimensionalites',
+                   torch.tensor([[0, 2, 4]]),
+                   torch.tensor([[0, 1, 0, 2]]),
+                   values([1, 2, 3, 4]),
+                   shape((2, 3)),
+                   r'values must have dimensionality > sum of batch and block dimensionalities \(=1 \+ 0\) but got 1')
+        else:
+            yield ('indices and values mismatch of dimensionalites',
+                   torch.tensor([[0, 2, 4]]),
+                   torch.tensor([[0, 1, 0, 2]]),
+                   values([1, 2, 3, 4]),
+                   shape((2, 3)),
+                   r'values must have dimensionality > sum of batch and block dimensionalities \(=1 \+ 2\) but got 3')
+
+        yield ('invalid size',
+               torch.tensor([0, 2, 4]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               (2,),
+               r'tensor dimensionality must be sum of batch, base, and dense dimensionalites \(=0 \+ 2 \+ 0\) but got 1')
+
+        yield ('invalid batchsize',
+               torch.tensor([[0, 2, 4]]),
+               torch.tensor([[0, 1, 0, 2]]),
+               values([[1, 2, 3, 4]]),
+               shape((2, 2, 3), 1),
+               r'all batch dimensions of compressed_indices \(=\[1\]\), plain_indices \(=\[1\]\), '
+               r'and values \(=\[1\]\) must be equal to tensor batch dimensions \(=\[2\]\)')
+
+        if layout is torch.sparse_bsr:
+            yield ('invalid blocksize',
+                   torch.tensor([0, 2, 4]),
+                   torch.tensor([0, 1, 0, 2]),
+                   torch.tensor([[[1, 11]], [[2, 22]], [[3, 33]], [[4, 33]]]),
+                   shape((2, 3)),
+                   r'tensor shape\[1\] \(=3\) must be divisible with blocksize\[1\] \(=2\) as defined by values shape')
+
+        if layout is torch.sparse_bsc:
+            yield ('invalid blocksize',
+                   torch.tensor([0, 2, 4]),
+                   torch.tensor([0, 1, 0, 2]),
+                   torch.tensor([[[1, 11]], [[2, 22]], [[3, 33]], [[4, 33]]]),
+                   shape((3, 2)),
+                   r'tensor shape\[1\] \(=3\) must be divisible with blocksize\[1\] \(=2\) as defined by values shape')
+
+        yield ('invalid compressed_indices shape',
+               torch.tensor([0, 2, 3, 4]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'compressed_indices.shape\[-1\] must be equal to the number of compressed_indices_names \+ 1 \(=3\), but got 4')
+
+        yield ('invalid compressed_indices shape',
+               torch.tensor([0, 2, 4]),
+               torch.tensor([0, 1, 0, 1, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'plain_indices.shape\[-1\] must be equal to nnz \(=4\) as defined by values.shape\[0\], but got 5')
+
+        yield ('compressed/plain_indices mismatch of dtype',
+               torch.tensor([0, 2, 4], dtype=torch.int32),
+               torch.tensor([0, 1, 0, 2], dtype=torch.int64),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'compressed_indices and plain_indices must have the same dtype, bot got Int and Long, respectively')
+
+        yield ('invalid compressed/plain_indices dtype',
+               torch.tensor([0, 2, 4], dtype=torch.int16),
+               torch.tensor([0, 1, 0, 2], dtype=torch.int16),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'compressed_indices and plain_indices dtype must be Int or Long, but got Short')
+
+        yield ('invalid compressed_indices[0]', torch.tensor([1, 2, 4]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'compressed_indices\[0\] \(=1\) == 0 is unsatisfied')
+
+        yield ('invalid compressed_indices[-1]',
+               torch.tensor([0, 2, 5]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'compressed_indices\[2\] \(=5\) <= nnz \(=4\) is unsatisfied')
+
+        yield ('invalid compressed_indices.diff(dim=-1)',
+               torch.tensor([0, 0, 4]),
+               torch.tensor([0, 1, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'compressed_indices\[2\] \(=4\) - compressed_indices\[1\] \(=0\) <= '
+               r'number of plain_indices_names \(=3\) is unsatisfied')
+
+        yield ('invalid max(plain_indices)',
+               torch.tensor([0, 2, 4]),
+               torch.tensor([0, 1, 0, 3]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'plain_indices\[3\] \(=3\) is out of range \(0, 3\)')
+
+        yield ('non-coalesced',
+               torch.tensor([0, 2, 4]),
+               torch.tensor([1, 0, 0, 2]),
+               values([1, 2, 3, 4]),
+               shape((2, 3)),
+               r'plain_indices must be ordered sequence of distinct integers but plain_indices\[0\] \(=1\) < '
+               r'plain_indices\[1\] \(=0\) is unsatisfied')
+
+        if TEST_CUDA:
+            yield ('indices and values mismatch of device',
+                   torch.tensor([0, 2, 4]),
+                   torch.tensor([0, 1, 0, 1]),
+                   values([1, 2, 3, 4], device='cuda'),
+                   shape((2, 3)),
+                   r'device of compressed_indices \(=cpu\) must match device of values \(=cuda:0\)')
+            yield ('compressed_indices and values mismatch of device',
+                   torch.tensor([0, 2, 4], device='cuda'),
+                   torch.tensor([0, 1, 0, 1]),
+                   values([1, 2, 3, 4]),
+                   shape((2, 3)),
+                   r'device of compressed_indices \(=cuda:0\) must match device of values \(=cpu\)')
+            yield ('compressed/plain_indices mismatch of device',
+                   torch.tensor([0, 2, 4], device='cuda'),
+                   torch.tensor([0, 1, 0, 1]),
+                   values([1, 2, 3, 4], device='cuda'),
+                   shape((2, 3)),
+                   r'device of compressed_indices \(=cuda:0\) must match device of plain_indices \(=cpu\)')
+
+    @skipMeta
+    @onlyCPU
+    @all_sparse_compressed_layouts()
+    @parametrize('target', [subtest('validate_sparse_compressed_tensor_args'),
+                            subtest('sparse_compressed_tensor'),
+                            subtest('sparse_compressed_tensor_no_size')])
+    def test_invalid_input(self, layout, target):
+        for label, compressed_indices, plain_indices, values, size, errmsg in self._generate_invalid_input(layout):
+            if layout is torch.sparse_bsr:
+                errmsg = errmsg.replace('compressed_indices_name', 'row block').replace('plain_indices_name', 'column block')
+            elif layout is torch.sparse_bsc:
+                errmsg = errmsg.replace('compressed_indices_name', 'column block').replace('plain_indices_name', 'row block')
+            elif layout is torch.sparse_csr:
+                errmsg = errmsg.replace('compressed_indices_name', 'row').replace('plain_indices_name', 'column')
+            elif layout is torch.sparse_csc:
+                errmsg = errmsg.replace('compressed_indices_name', 'column').replace('plain_indices_name', 'row')
+            if layout in {torch.sparse_csr, torch.sparse_bsr}:
+                errmsg = errmsg.replace('compressed_indices', 'crow_indices').replace('plain_indices', 'col_indices')
+            else:
+                errmsg = errmsg.replace('compressed_indices', 'ccol_indices').replace('plain_indices', 'row_indices')
+
+            if target == 'sparse_compressed_tensor_no_size' and label in {
+                    'invalid size', 'invalid batchsize', 'invalid compressed_indices shape', 'invalid max(plain_indices)',
+                    'invalid blocksize'}:
+                # Skip invalid size input as a valid size is estimated for other inputs
+                continue
+
+            with self.assertRaisesRegex(RuntimeError, errmsg):
+                if target == 'validate_sparse_compressed_tensor_args':
+                    torch._validate_sparse_compressed_tensor_args(compressed_indices, plain_indices, values, size, layout)
+                elif target == 'sparse_compressed_tensor':
+                    torch.sparse_compressed_tensor(compressed_indices, plain_indices, values, size, layout=layout)
+                elif target == 'sparse_compressed_tensor_no_size':
+                    torch.sparse_compressed_tensor(compressed_indices, plain_indices, values, layout=layout)
+                else:
+                    raise NotImplementedError(target)
+
+
 class TestSparseCSR(TestCase):
 
     def test_csr_stride(self):
@@ -621,161 +915,6 @@ def test_resize_errors(self, device, dtype):
                 new_shape = (2, 2)
                 a.resize_(new_shape)
 
-    def test_factory_type_invariants_check(self, device):
-        with self.assertRaisesRegex(RuntimeError, "both crow_indices and col_indices should have the same type."):
-            torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int64),
-                                    torch.tensor([0, 1, 0, 1], dtype=torch.int32),
-                                    torch.tensor([1, 2, 3, 4]),
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError, "crow_indices and col_indices must be an int32 or int64 type"):
-            torch.sparse_csr_tensor(torch.tensor([0, 2, 4], dtype=torch.int16),
-                                    torch.tensor([0, 1, 0, 1], dtype=torch.int16),
-                                    torch.tensor([1, 2, 3, 4]),
-                                    device=device)
-
-    def test_factory_layout_invariants_check(self, device):
-        with self.assertRaisesRegex(RuntimeError, "expected values to be a strided and contiguous tensor"):
-            values = torch.tensor([1.], device=device).expand(4,)
-            torch.sparse_csr_tensor(torch.tensor([0, 2, 4], device=device),
-                                    torch.tensor([0, 1, 0, 1], device=device),
-                                    values)
-
-        with self.assertRaisesRegex(RuntimeError, "expected col_indices to be a strided and contiguous tensor"):
-            col_indices = torch.tensor([0], device=device).expand(4,)
-            torch.sparse_csr_tensor(torch.tensor([0, 2, 4]),
-                                    col_indices,
-                                    torch.tensor([1, 2, 3, 4]))
-
-        with self.assertRaisesRegex(RuntimeError, "expected crow_indices to be a strided and contiguous tensor"):
-            crow_indices = torch.arange(6, device=device)
-            torch.sparse_csr_tensor(crow_indices[::2],
-                                    torch.tensor([0, 1, 0, 1], device=device),
-                                    torch.tensor([1, 2, 3, 4]))
-
-    def test_factory_shape_invariants_check(self, device):
-        crow_indices = torch.tensor([0, 2, 4], device=device)
-        col_indices = torch.tensor([0, 1, 0, 1], device=device)
-        values = torch.tensor([1, 2, 3, 4], device=device)
-        size = (2, 10)
-        torch.sparse_csr_tensor(crow_indices, col_indices, values, size, device=device)
-
-        with self.assertRaisesRegex(RuntimeError, r"size of a batched CSR tensor must have length >= 2, but got: 1"):
-            torch.sparse_csr_tensor(crow_indices, col_indices, values,
-                                    size=(2,),
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError, r"crow_indices must have dim >= 1 but got crow_indices\.dim\(\)\ = 0"):
-            torch.sparse_csr_tensor(torch.zeros((), device=device, dtype=torch.int64),
-                                    col_indices,
-                                    values,
-                                    size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError, r"col_indices must have dim >= 1 but got col_indices\.dim\(\)\ = 0"):
-            torch.sparse_csr_tensor(crow_indices,
-                                    torch.zeros((), device=device, dtype=torch.int64),
-                                    values,
-                                    size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError, r"values must have dim >= 1 but got values\.dim\(\)\ = 0"):
-            torch.sparse_csr_tensor(crow_indices,
-                                    col_indices,
-                                    torch.zeros((), device=device, dtype=torch.int64),
-                                    size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    r"crow_indices\.size\(-1\) must be equal to size\[-2\] \+ 1 \(that is 2\), but got: 3"):
-            torch.sparse_csr_tensor(crow_indices, col_indices, values, (1, 1),
-                                    device=device)
-
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    r"number of dimensions of crow_indices and col_indices must be the same"):
-            torch.sparse_csr_tensor(crow_indices, col_indices.repeat(2, 1), values, size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    r"non-zero dense dimensions \(=1\) is not supported"):
-            torch.sparse_csr_tensor(crow_indices, col_indices, values.repeat(2, 1), size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    r"number of dimensions of indices must be one less"):
-            torch.sparse_csr_tensor(crow_indices.repeat(2, 1), col_indices.repeat(2, 1), values.repeat(2, 1), size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    r"all batch dimensions of the provided size \(\[2\]\), indices \(\[2\], \[3\]\),"
-                                    r" and values \(\[4\]\) must be the same"):
-            torch.sparse_csr_tensor(crow_indices.repeat(2, 1), col_indices.repeat(3, 1), values.repeat(4, 1), (2, 2, 10),
-                                    device=device)
-
-    def test_factory_indices_invariants_check(self, device):
-        crow_indices = [0, 2, 4]
-        col_indices = [0, 1, 0, 1]
-        values = [1, 2, 3, 4]
-        size = (2, 10)
-        with self.assertRaisesRegex(RuntimeError, "0th value of crow_indices must be 0."):
-            torch.sparse_csr_tensor(torch.tensor([-1, 0, 4]), torch.tensor(col_indices), torch.tensor(values), size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    "last value of crow_indices should be equal to the length of col_indices."):
-            torch.sparse_csr_tensor(torch.tensor([0, 2, 5]), torch.tensor(col_indices), torch.tensor(values), size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError,
-                                    r"at position i \= 2," +
-                                    r" the condition crow_indices\[i - 1\] <\= crow_indices\[i\] fails"):
-            torch.sparse_csr_tensor(torch.tensor([0, 5, 4]), torch.tensor(col_indices), torch.tensor(values), size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError, r"col_indices\.min\(\) should be greater or equal to zero"):
-            torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, -1, 0, 1]), torch.tensor(values), size,
-                                    device=device)
-
-        with self.assertRaisesRegex(RuntimeError, r"size\[-1\] should be greater than col_indices\.max\(\)"):
-            torch.sparse_csr_tensor(torch.tensor(crow_indices), torch.tensor([0, 11, 0, 1]), torch.tensor(values), size,
-                                    device=device)
-
-    @onlyCUDA
-    @dtypes(*all_types_and_complex_and(torch.half, torch.bool, torch.bfloat16))
-    def test_factory_device_type_inference(self, device, dtype):
-        cpu_cuda = ('cpu', 'cuda')
-        cpu_cuda_none = cpu_cuda + (None,)
-        for crow_indices_device, col_indices_device, values_device, device in itertools.product(cpu_cuda,
-                                                                                                cpu_cuda,
-                                                                                                cpu_cuda,
-                                                                                                cpu_cuda_none):
-            for index_dtype in [torch.int32, torch.int64]:
-                crow_indices = torch.tensor([0, 2, 4], dtype=index_dtype, device=crow_indices_device)
-                col_indices = torch.tensor([0, 1, 0, 1], dtype=index_dtype, device=col_indices_device)
-                values = torch.tensor([1, 2, 3, 4], dtype=dtype, device=values_device)
-                if device is None and (crow_indices_device != col_indices_device or
-                                       crow_indices_device != values_device):
-                    with self.assertRaises(RuntimeError):
-                        torch.sparse_csr_tensor(crow_indices,
-                                                col_indices,
-                                                values,
-                                                size=(2, 10),
-                                                device=device)
-                else:
-                    t = torch.sparse_csr_tensor(crow_indices,
-                                                col_indices,
-                                                values,
-                                                size=(2, 10),
-                                                device=device)
-                    should_be_cuda = (device == 'cuda' or (device is None and values_device == 'cuda'))
-                    self.assertEqual(should_be_cuda, t.is_cuda)
-                    t.crow_indices().dtype == index_dtype
-                    t.col_indices().dtype == index_dtype
-                    t.values().dtype == dtype
-                    t.crow_indices().device == t.values().device
-                    t.col_indices().device == t.values().device
-
     @dtypes(*all_types_and_complex_and(torch.half, torch.bool, torch.bfloat16))
     def test_sparse_csr_from_dense(self, device, dtype):
         dense = torch.tensor([[4, 5, 0], [0, 0, 0], [1, 0, 0]], dtype=dtype, device=device)
diff --git a/test/test_testing.py b/test/test_testing.py
index 5db1318725766..4b23b6b0fb4ed 100644
--- a/test/test_testing.py
+++ b/test/test_testing.py
@@ -1134,7 +1134,7 @@ def test_matching(self):
 
     def test_mismatching_crow_indices_msg(self):
         actual_crow_indices = (0, 1, 2)
-        actual_col_indices = (1, 0)
+        actual_col_indices = (0, 1)
         actual_values = (1, 2)
         actual = torch.sparse_csr_tensor(actual_crow_indices, actual_col_indices, actual_values, size=(2, 2))
 
@@ -1192,7 +1192,7 @@ def test_matching(self):
 
     def test_mismatching_ccol_indices_msg(self):
         actual_ccol_indices = (0, 1, 2)
-        actual_row_indices = (1, 0)
+        actual_row_indices = (0, 1)
         actual_values = (1, 2)
         actual = torch.sparse_csc_tensor(actual_ccol_indices, actual_row_indices, actual_values, size=(2, 2))
 
@@ -1250,7 +1250,7 @@ def test_matching(self):
 
     def test_mismatching_crow_indices_msg(self):
         actual_crow_indices = (0, 1, 2)
-        actual_col_indices = (1, 0)
+        actual_col_indices = (0, 1)
         actual_values = ([[1]], [[2]])
         actual = torch.sparse_bsr_tensor(actual_crow_indices, actual_col_indices, actual_values, size=(2, 2))
 
@@ -1308,7 +1308,7 @@ def test_matching(self):
 
     def test_mismatching_ccol_indices_msg(self):
         actual_ccol_indices = (0, 1, 2)
-        actual_row_indices = (1, 0)
+        actual_row_indices = (0, 1)
         actual_values = ([[1]], [[2]])
         actual = torch.sparse_bsc_tensor(actual_ccol_indices, actual_row_indices, actual_values, size=(2, 2))