I has test all javacpp -pytorch dataset and dataloader ,but JavaStateful meet error?

[mullerhai]

these object JavaStatefulTensorDataLoader JavaStatefulTensorDataset JavaStatefulTensorDataLoader JavaStatefulTensorDataset I don't know how to use ,when I try to run two example ,but console is fault ,other dataset can use

import org.bytedeco.javacpp.*;
import org.bytedeco.pytorch.*;
import org.bytedeco.pytorch.DataLoaderOptions;
import org.bytedeco.pytorch.RandomSampler; //{ ExampleIterator, ExampleVectorOptional,DataLoaderOptions, FullDataLoaderOptions, InputArchive, OutputArchive, SizeTOptional, SizeTVectorOptional, T_TensorT_TensorTensor_T_T, T_TensorTensor_T, T_TensorTensor_TOptional, TensorMapper, TensorVector, TransformerImpl, TransformerOptions, kCircular, kGELU, kReflect, kReplicate, kZeros, ChunkBatchDataset, ChunkRandomDataLoader, RandomSampler , SequentialSampler };

public class TestChunkDatas {

    public static void main(String[] args) throws Exception {
        try (PointerScope scope = new PointerScope()) {
            long batch_size = 10;
            long prefetch_count = 1;

            JavaStatefulTensorDataset ds = new JavaStatefulTensorDataset(){
                TensorExampleVector ex = new TensorExampleVector(
                        new TensorExample(Tensor.create(10.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(15.0, 30.0, 50.0, 80.0, 300.0)),
                        new TensorExample(Tensor.create(20.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(35.0, 30.0, 50.0, 80.0, 300.0)),
                        new TensorExample(Tensor.create(40.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(55.0, 30.0, 50.0, 80.0, 300.0)),
                        new TensorExample(Tensor.create(60.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(75.0, 30.0, 50.0, 80.0, 300.0))

                );


                @Override
                public TensorExampleVectorOptional get_batch(long size) {
                    return new TensorExampleVectorOptional(ex);
                }

                @Override
                public void reset() {
                    super.reset();
                }
                //                @Override
//                public TensorExampleVector get_batch(long size) {
//                    return ex;
//                }
//
                @Override
                public SizeTOptional size() {
                    return new SizeTOptional(ex.size());
                }
            };

            SequentialSampler sampler = new SequentialSampler(0);
            DataLoaderOptions opts = new DataLoaderOptions(2);
            opts.workers().put(5);
            JavaStatefulTensorDataLoader data_loader = new JavaStatefulTensorDataLoader(ds, opts); //.map(new ExampleStack());
//            SequentialSampler sampler = new SequentialSampler(0);
//            ChunkMapTensorDataset data_set = new ChunkSharedTensorBatchDataset(
//                    new ChunkTensorDataset(data_reader, sampler, sampler,
//                            new ChunkDatasetOptions(prefetch_count, batch_size))).map(new TensorExampleStack());
//            ChunkRandomTensorDataLoader data_loader = new ChunkRandomTensorDataLoader(
//                    data_set, new DataLoaderOptions(batch_size));
            for (int epoch = 1; epoch <= 10; ++epoch) {
                for (TensorExampleVectorIterator it = data_loader.begin(); !it.equals(data_loader.end()); it = it.increment()) {
                    TensorExampleVector batch = it.access();
                    System.out.println(batch );
                }
            }
        }
    }

    public static void mainrty(String[] args) throws Exception {
        try (PointerScope scope = new PointerScope()) {
            long batch_size = 10;
            long prefetch_count = 1;

            JavaStatefulTensorDataset ds = new JavaStatefulTensorDataset(){


                TensorExampleVector ex = new TensorExampleVector(
                        new TensorExample(Tensor.create(10.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(15.0, 30.0, 50.0, 80.0, 300.0)),
                        new TensorExample(Tensor.create(20.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(35.0, 30.0, 50.0, 80.0, 300.0)),
                        new TensorExample(Tensor.create(40.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(55.0, 30.0, 50.0, 80.0, 300.0)),
                        new TensorExample(Tensor.create(60.0, 20.0, 50.0, 80.0, 100.0)),
                        new TensorExample(Tensor.create(75.0, 30.0, 50.0, 80.0, 300.0))

                );

                public TensorExample get(long index) {
                    return ex.get(index);
//                    return super.get(index);
                }

                @Override
                public TensorExampleVectorOptional get_batch(long size) {
                    return new TensorExampleVectorOptional(ex );
                }

                @Override
                public void reset() {
                    super.reset();
                }

                @Override
                public SizeTOptional size() {
                    return new SizeTOptional(ex.size());
                }
            };

            SequentialSampler sampler = new SequentialSampler(0);
            DataLoaderOptions opts = new DataLoaderOptions(2);
            opts.workers().put(5);
            JavaStatefulTensorDataLoader data_loader = new JavaStatefulTensorDataLoader(ds,  opts); //.map(new ExampleStack()); new DistributedRandomSampler(ds.size().get()),
//            SequentialSampler sampler = new SequentialSampler(0);
//            ChunkMapTensorDataset data_set = new ChunkSharedTensorBatchDataset(
//                    new ChunkTensorDataset(data_reader, sampler, sampler,
//                            new ChunkDatasetOptions(prefetch_count, batch_size))).map(new TensorExampleStack());
//            ChunkRandomTensorDataLoader data_loader = new ChunkRandomTensorDataLoader(
//                    data_set, new DataLoaderOptions(batch_size));
            for (int epoch = 1; epoch <= 10; ++epoch) {
                for (TensorExampleVectorIterator it = data_loader.begin(); !it.equals(data_loader.end()); it = it.increment()) {
                    TensorExampleVector batch = it.access();
                    System.out.println("hello stateful");
                    System.out.println(batch );
                }
            }
        }
    }
}

console

Exception in thread "main" java.lang.RuntimeException: java.lang.RuntimeException: Cannot call pure virtual function javacpp::StatefulDataset<torch::Tensor,torch::data::example::NoTarget>::reset().
	at org.bytedeco.pytorch.JavaStatefulTensorDataLoaderBase.begin(Native Method)
	at example.TestChunkDatas.main(TestChunkDatas.java:60) 

[mullerhai]

if reset() not invoke super.reset ,also will meet error

               @Override
                public void reset() {
//                    //super.reset();
                }

console

C:\Users\jeffsyry\.jdks\openjdk-23.0.1\bin\java.exe "-javaagent:C:\Program Files\JetBrains\IntelliJ IDEA 2024.3\lib\idea_rt.jar=49591:C:\Program Files\JetBrains\IntelliJ IDEA 2024.3\bin" -Dfile.encoding=UTF-8 -Dsun.stdout.encoding=UTF-8 -Dsun.stderr.encoding=UTF-8 -classpath E:\gitcode\core\target\scala-3.6.2\classes;D:\Coursier\https\repo1.maven.org\maven2\org\scala-lang\scala3-library_3\3.6.2\scala3-library_3-3.6.2.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\javacpp\1.5.11\javacpp-1.5.11.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\javacpp\1.5.11\javacpp-1.5.11-windows-x86_64.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\pytorch\2.5.1-1.5.11\pytorch-2.5.1-1.5.11.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\pytorch\2.5.1-1.5.11\pytorch-2.5.1-1.5.11-windows-x86_64-gpu.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\pytorch\2.5.1-1.5.11\pytorch-2.5.1-1.5.11-windows-x86_64.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\openblas\0.3.28-1.5.11\openblas-0.3.28-1.5.11.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\openblas\0.3.28-1.5.11\openblas-0.3.28-1.5.11-windows-x86_64.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\cuda\12.6-9.5-1.5.11\cuda-12.6-9.5-1.5.11.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\cuda\12.6-9.5-1.5.11\cuda-12.6-9.5-1.5.11-windows-x86_64-redist.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\mkl\2025.0-1.5.11\mkl-2025.0-1.5.11.jar;D:\Coursier\https\repo1.maven.org\maven2\org\bytedeco\mkl\2025.0-1.5.11\mkl-2025.0-1.5.11-windows-x86_64.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\spire_3\0.18.0\spire_3-0.18.0.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\shapeless3-typeable_3\3.3.0\shapeless3-typeable_3-3.3.0.jar;D:\Coursier\https\repo1.maven.org\maven2\com\lihaoyi\os-lib_3\0.9.1\os-lib_3-0.9.1.jar;D:\Coursier\https\repo1.maven.org\maven2\com\lihaoyi\sourcecode_3\0.3.0\sourcecode_3-0.3.0.jar;D:\Coursier\https\repo1.maven.org\maven2\dev\dirs\directories\26\directories-26.jar;D:\Coursier\https\repo1.maven.org\maven2\org\scala-lang\scala-library\2.13.15\scala-library-2.13.15.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\spire-macros_3\0.18.0\spire-macros_3-0.18.0.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\spire-platform_3\0.18.0\spire-platform_3-0.18.0.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\spire-util_3\0.18.0\spire-util_3-0.18.0.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\algebra_3\2.8.0\algebra_3-2.8.0.jar;D:\Coursier\https\repo1.maven.org\maven2\com\lihaoyi\geny_3\1.0.0\geny_3-1.0.0.jar;D:\Coursier\https\repo1.maven.org\maven2\org\typelevel\cats-kernel_3\2.8.0\cats-kernel_3-2.8.0.jar 





example.TestChunkDatas
Exception in thread "main" java.lang.RuntimeException: Attempted to get a new DataLoader iterator while another iterator is not yet exhausted
Exception raised from begin at D:\a\javacpp-presets\javacpp-presets\pytorch\cppbuild\windows-x86_64-gpu\include\torch\csrc\api\include\torch/data/dataloader/base.h:62 (most recent call first):
00007FFEE38383C9 <unknown symbol address> c10.dll!<unknown symbol> [<unknown file> @ <unknown line number>]
00007FFEE3836BEA <unknown symbol address> c10.dll!<unknown symbol> [<unknown file> @ <unknown line number>]
00007FFEDA715A57 <unknown symbol address> jnitorch.dll!<unknown symbol> [<unknown file> @ <unknown line number>]
000002028753D207 <unknown symbol address> !<unknown symbol> [<unknown file> @ <unknown line number>]

	at org.bytedeco.pytorch.JavaStatefulTensorDataLoaderBase.begin(Native Method)
	at example.TestChunkDatas.main(TestChunkDatas.java:61)

Process finished with exit code 1

[mullerhai]

Now I want to say some things, some dataset also have error ,ChunkStatefulDataset ,ChunkStatefulTensorDataset I not know how to use it , because of it not have ChunkStatefulDataLoader? other Dataloader can not use .why ?
could you give me some example to use them ? thanks @saudet