From b9fb7df31e0ee07bf6a113bbb652d30924ce863a Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 26 Sep 2023 00:47:21 +0300 Subject: [PATCH 01/57] add all files to check code style --- .../itmo/test/kachmareugene/DaoFactory.java | 36 ++++++++ .../itmo/test/kachmareugene/InMemoryDao.java | 86 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java new file mode 100644 index 000000000..672217838 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -0,0 +1,36 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +@ru.vk.itmo.test.DaoFactory +public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { + @Override + public String toString(MemorySegment memorySegment) { + return new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); + } + + @Override + public MemorySegment fromString(String data) { + if (data == null) { + return null; + } + return Arena.ofAuto().allocateArray(ValueLayout.JAVA_BYTE, data.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public Entry fromBaseEntry(Entry baseEntry) { + return baseEntry; + } + + @Override + public Dao> createDao() { + return new InMemoryDao(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java new file mode 100644 index 000000000..3f200e783 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -0,0 +1,86 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.*; +import java.util.concurrent.ConcurrentSkipListMap; + +public class InMemoryDao implements Dao> { + + private final SortedMap mp; + public InMemoryDao() { + this.mp = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + Iterator> iterator = null; + + if (from == null && to == null) { + iterator = mp.entrySet().iterator(); + } + if (from == null && to != null) { + iterator = mp.headMap(to).entrySet().iterator(); + } + + if (from != null && to == null) { + iterator = mp.tailMap(from).entrySet().iterator(); + } + if (from != null && to != null) { + iterator = mp.subMap(from, to).entrySet().iterator(); + } + + + return new Iterator>() { + Iterator> iterator; + public Iterator> init(Iterator> it) { + this.iterator = it; + return this; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Entry next() { + var rawEntry = iterator.next(); + return new BaseEntry<>(rawEntry.getKey(), rawEntry.getValue()); + } + }.init(iterator); + } + + + @Override + public Entry get(MemorySegment key) { + return new BaseEntry<>(key, mp.get(key)); + } + + @Override + public void upsert(Entry entry) { + mp.put(entry.key(), entry.value()); + } + + private static class MemorySegmentComparator implements Comparator { + + @Override + public int compare(MemorySegment segment1, MemorySegment segment2) { + if (segment1.byteSize() != segment2.byteSize()) { + return Long.compare(segment1.byteSize(), segment2.byteSize()); + } + + long firstDiffByte = segment1.mismatch(segment2); + + if (firstDiffByte == -1) { + return 0; + } + return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, firstDiffByte), + segment2.get(ValueLayout.JAVA_BYTE, firstDiffByte)); + } + } +} From 63cd28065968cfd96ef5045102d2ed32bdb5d7f4 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 26 Sep 2023 12:03:47 +0300 Subject: [PATCH 02/57] optimize map values, if symplified --- .../itmo/test/kachmareugene/DaoFactory.java | 1 - .../itmo/test/kachmareugene/InMemoryDao.java | 44 ++++++++----------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 672217838..8b65fa9bd 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -5,7 +5,6 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 3f200e783..b91e60078 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -1,43 +1,39 @@ package ru.vk.itmo.test.kachmareugene; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.util.*; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map; +import java.util.SortedMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDao implements Dao> { - private final SortedMap mp; + private final SortedMap> mp; public InMemoryDao() { this.mp = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); } - @Override public Iterator> get(MemorySegment from, MemorySegment to) { - Iterator> iterator = null; + SortedMap> dataSlice; if (from == null && to == null) { - iterator = mp.entrySet().iterator(); - } - if (from == null && to != null) { - iterator = mp.headMap(to).entrySet().iterator(); + dataSlice = mp; + } else if (from == null) { + dataSlice = mp.headMap(to); + } else if (to == null) { + dataSlice = mp.tailMap(from); + } else { + dataSlice = mp.subMap(from, to); } - if (from != null && to == null) { - iterator = mp.tailMap(from).entrySet().iterator(); - } - if (from != null && to != null) { - iterator = mp.subMap(from, to).entrySet().iterator(); - } - - return new Iterator>() { - Iterator> iterator; - public Iterator> init(Iterator> it) { + Iterator>> iterator; + public Iterator> init(Iterator>> it) { this.iterator = it; return this; } @@ -49,21 +45,19 @@ public boolean hasNext() { @Override public Entry next() { - var rawEntry = iterator.next(); - return new BaseEntry<>(rawEntry.getKey(), rawEntry.getValue()); + return iterator.next().getValue(); } - }.init(iterator); + }.init(dataSlice.entrySet().iterator()); } - @Override public Entry get(MemorySegment key) { - return new BaseEntry<>(key, mp.get(key)); + return mp.get(key); } @Override public void upsert(Entry entry) { - mp.put(entry.key(), entry.value()); + mp.put(entry.key(), entry); } private static class MemorySegmentComparator implements Comparator { From c25d10aedd8d8ba0138e593e569b5ba97515e47a Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 26 Sep 2023 12:09:29 +0300 Subject: [PATCH 03/57] add 2 blank lines --- src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index b91e60078..87dd1d608 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -14,9 +14,11 @@ public class InMemoryDao implements Dao> { private final SortedMap> mp; + public InMemoryDao() { this.mp = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); } + @Override public Iterator> get(MemorySegment from, MemorySegment to) { SortedMap> dataSlice; From e080066f5eacc842801cf229d76465095f884ae4 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 26 Sep 2023 12:30:16 +0300 Subject: [PATCH 04/57] remove redundant iterator anonumus class --- .../itmo/test/kachmareugene/InMemoryDao.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 87dd1d608..d47f6ef77 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -7,7 +7,6 @@ import java.lang.foreign.ValueLayout; import java.util.Comparator; import java.util.Iterator; -import java.util.Map; import java.util.SortedMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -33,23 +32,7 @@ public Iterator> get(MemorySegment from, MemorySegment to) dataSlice = mp.subMap(from, to); } - return new Iterator>() { - Iterator>> iterator; - public Iterator> init(Iterator>> it) { - this.iterator = it; - return this; - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public Entry next() { - return iterator.next().getValue(); - } - }.init(dataSlice.entrySet().iterator()); + return dataSlice.values().iterator(); } @Override From 05a9ba0e4af891fd8f7d8ab39ffb38f3ac0ae365 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 27 Sep 2023 14:42:03 +0300 Subject: [PATCH 05/57] fixed comparator --- .../ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index d47f6ef77..65ad3dfc6 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -10,6 +10,8 @@ import java.util.SortedMap; import java.util.concurrent.ConcurrentSkipListMap; +import static java.lang.Long.min; + public class InMemoryDao implements Dao> { private final SortedMap> mp; @@ -49,15 +51,17 @@ private static class MemorySegmentComparator implements Comparator Date: Tue, 3 Oct 2023 19:45:52 +0300 Subject: [PATCH 06/57] init commit --- .../itmo/test/kachmareugene/DaoFactory.java | 12 ++- .../itmo/test/kachmareugene/InMemoryDao.java | 101 +++++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 8b65fa9bd..cc9f17d8e 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -1,14 +1,15 @@ package ru.vk.itmo.test.kachmareugene; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.lang.foreign.Arena; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@ru.vk.itmo.test.DaoFactory +@ru.vk.itmo.test.DaoFactory(stage = 2) public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -20,7 +21,7 @@ public MemorySegment fromString(String data) { if (data == null) { return null; } - return Arena.ofAuto().allocateArray(ValueLayout.JAVA_BYTE, data.getBytes(StandardCharsets.UTF_8)); + return MemorySegment.ofArray(data.getBytes(StandardCharsets.UTF_8)); } @Override @@ -32,4 +33,9 @@ public Entry fromBaseEntry(Entry baseEntry) { public Dao> createDao() { return new InMemoryDao(); } + + @Override + public Dao> createDao(Config config) throws IOException { + return new InMemoryDao(config.basePath()); + } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 65ad3dfc6..f9a4aa7c4 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -1,23 +1,42 @@ package ru.vk.itmo.test.kachmareugene; +import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Comparator; import java.util.Iterator; +import java.util.Set; import java.util.SortedMap; import java.util.concurrent.ConcurrentSkipListMap; import static java.lang.Long.min; +import static java.nio.file.StandardOpenOption.*; public class InMemoryDao implements Dao> { - private final SortedMap> mp; + + private final Comparator memorySegmentComparatorImpl = new MemorySegmentComparator(); + private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); + + private final Path ssTablesDir; + private final String ssTableName = "ssTable.txt"; public InMemoryDao() { - this.mp = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); + this.ssTablesDir = null; + } + public InMemoryDao(Path path) { + this.ssTablesDir = path; } @Override @@ -37,9 +56,42 @@ public Iterator> get(MemorySegment from, MemorySegment to) return dataSlice.values().iterator(); } + private MemorySegment searchKeyInFile(MemorySegment mapped, MemorySegment key, long maxSize) { + long offset = 0L; + long kSize, vSize; + + while (offset < maxSize) { + offset = alignmentBy(offset, 8); + kSize = mapped.get(ValueLayout.JAVA_LONG, offset); + offset += Long.BYTES; + vSize = mapped.get(ValueLayout.JAVA_LONG, offset); + offset += Long.BYTES; + + if (memorySegmentComparatorImpl.compare(key, mapped.asSlice(offset, kSize)) == 0) { + return mapped.asSlice(offset + kSize, vSize); + } + offset += kSize + vSize; + } + return null; + } + @Override public Entry get(MemorySegment key) { - return mp.get(key); + if (mp.containsKey(key)) { + return mp.get(key); + } + + // fixme check existence + if (ssTablesDir == null) { + return null; + } + Set options = Set.of(READ); + try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), options)) { + MemorySegment mapped = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), Arena.ofAuto()); + return new BaseEntry<>(key, searchKeyInFile(mapped, key, channel.size())); + } catch (IOException e) { + return null; + } } @Override @@ -66,4 +118,47 @@ public int compare(MemorySegment segment1, MemorySegment segment2) { segment2.get(ValueLayout.JAVA_BYTE, firstDiffByte)); } } + + + private long dumpSegmentSize(MemorySegment mapped, long size, long offset) { + long formattedOffset = alignmentBy(offset, 8); + mapped.set(ValueLayout.JAVA_LONG, formattedOffset, size); + return formattedOffset + Long.BYTES; + } + private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { + MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); + return offset + data.byteSize(); + } + + + private long alignmentBy(long data, long by) { + return data % by != 0 ? data + (by - data % by) : data; + } + @Override + public void close() throws IOException { + if (ssTablesDir == null) { + return; + } + Set options = Set.of(WRITE, READ, StandardOpenOption.CREATE); + try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), options)) { + long currOffset = 0L; + long allDataLen = 0L; + for (var kv : mp.values()) { + allDataLen += alignmentBy(kv.key().byteSize() + kv.value().byteSize() + 16, 8); + } + MemorySegment mappedSegment = channel.map( + FileChannel.MapMode.READ_WRITE, currOffset, allDataLen, Arena.ofAuto()); + + for (var kv : mp.values()) { + currOffset = dumpSegmentSize(mappedSegment, kv.key().byteSize(), currOffset); + currOffset = dumpSegmentSize(mappedSegment, kv.value().byteSize(), currOffset); + currOffset = dumpSegment(mappedSegment, kv.key(), currOffset); + currOffset = dumpSegment(mappedSegment, kv.value(), currOffset); + } + channel.force(true); + channel. + } finally { + mp.clear(); + } + } } From 05920556e71009a63371ef8865f68698dd5355ab Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 3 Oct 2023 19:53:54 +0300 Subject: [PATCH 07/57] remove statement --- src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index f9a4aa7c4..05fc33715 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -155,8 +155,7 @@ public void close() throws IOException { currOffset = dumpSegment(mappedSegment, kv.key(), currOffset); currOffset = dumpSegment(mappedSegment, kv.value(), currOffset); } - channel.force(true); - channel. + } finally { mp.clear(); } From 89313f9ab5aff3a65e444c95d458514bdc4f8872 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 3 Oct 2023 20:05:54 +0300 Subject: [PATCH 08/57] clean codestyle --- .../itmo/test/kachmareugene/InMemoryDao.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 05fc33715..af0d1860c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -4,9 +4,7 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.RandomAccessFile; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -25,16 +23,17 @@ public class InMemoryDao implements Dao> { - private final Comparator memorySegmentComparatorImpl = new MemorySegmentComparator(); - private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); + private final SortedMap> mp = + new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); private final Path ssTablesDir; - private final String ssTableName = "ssTable.txt"; + private final static String ssTableName = "ssTable.txt"; public InMemoryDao() { this.ssTablesDir = null; } + public InMemoryDao(Path path) { this.ssTablesDir = path; } @@ -58,7 +57,8 @@ public Iterator> get(MemorySegment from, MemorySegment to) private MemorySegment searchKeyInFile(MemorySegment mapped, MemorySegment key, long maxSize) { long offset = 0L; - long kSize, vSize; + long kSize; + long vSize; while (offset < maxSize) { offset = alignmentBy(offset, 8); @@ -81,12 +81,11 @@ public Entry get(MemorySegment key) { return mp.get(key); } - // fixme check existence if (ssTablesDir == null) { return null; } - Set options = Set.of(READ); - try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), options)) { + + try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), READ)) { MemorySegment mapped = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), Arena.ofAuto()); return new BaseEntry<>(key, searchKeyInFile(mapped, key, channel.size())); } catch (IOException e) { @@ -119,27 +118,28 @@ public int compare(MemorySegment segment1, MemorySegment segment2) { } } - private long dumpSegmentSize(MemorySegment mapped, long size, long offset) { long formattedOffset = alignmentBy(offset, 8); mapped.set(ValueLayout.JAVA_LONG, formattedOffset, size); return formattedOffset + Long.BYTES; } + private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); return offset + data.byteSize(); } - private long alignmentBy(long data, long by) { - return data % by != 0 ? data + (by - data % by) : data; + return data % by == 0 ? data : data + (by - data % by); } + @Override public void close() throws IOException { if (ssTablesDir == null) { return; } - Set options = Set.of(WRITE, READ, StandardOpenOption.CREATE); + + Set options = Set.of(WRITE, READ, CREATE); try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), options)) { long currOffset = 0L; long allDataLen = 0L; @@ -155,7 +155,6 @@ public void close() throws IOException { currOffset = dumpSegment(mappedSegment, kv.key(), currOffset); currOffset = dumpSegment(mappedSegment, kv.value(), currOffset); } - } finally { mp.clear(); } From cfe33088ca43dc75a1979b8f627716a10e03aeb2 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 3 Oct 2023 20:15:58 +0300 Subject: [PATCH 09/57] some clean, renaming --- .../itmo/test/kachmareugene/InMemoryDao.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index af0d1860c..edd262863 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -11,7 +11,6 @@ import java.nio.channels.FileChannel; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Comparator; import java.util.Iterator; import java.util.Set; @@ -19,7 +18,9 @@ import java.util.concurrent.ConcurrentSkipListMap; import static java.lang.Long.min; -import static java.nio.file.StandardOpenOption.*; +import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.WRITE; +import static java.nio.file.StandardOpenOption.CREATE; public class InMemoryDao implements Dao> { @@ -28,7 +29,7 @@ public class InMemoryDao implements Dao> { new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); private final Path ssTablesDir; - private final static String ssTableName = "ssTable.txt"; + private static final String SS_TABLE_NAME = "ssTable.txt"; public InMemoryDao() { this.ssTablesDir = null; @@ -57,20 +58,20 @@ public Iterator> get(MemorySegment from, MemorySegment to) private MemorySegment searchKeyInFile(MemorySegment mapped, MemorySegment key, long maxSize) { long offset = 0L; - long kSize; - long vSize; + long keyByteSize; + long valueByteSize; while (offset < maxSize) { offset = alignmentBy(offset, 8); - kSize = mapped.get(ValueLayout.JAVA_LONG, offset); + keyByteSize = mapped.get(ValueLayout.JAVA_LONG, offset); offset += Long.BYTES; - vSize = mapped.get(ValueLayout.JAVA_LONG, offset); + valueByteSize = mapped.get(ValueLayout.JAVA_LONG, offset); offset += Long.BYTES; - if (memorySegmentComparatorImpl.compare(key, mapped.asSlice(offset, kSize)) == 0) { - return mapped.asSlice(offset + kSize, vSize); + if (memorySegmentComparatorImpl.compare(key, mapped.asSlice(offset, keyByteSize)) == 0) { + return mapped.asSlice(offset + keyByteSize, valueByteSize); } - offset += kSize + vSize; + offset += keyByteSize + valueByteSize; } return null; } @@ -85,7 +86,7 @@ public Entry get(MemorySegment key) { return null; } - try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), READ)) { + try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(SS_TABLE_NAME), READ)) { MemorySegment mapped = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), Arena.ofAuto()); return new BaseEntry<>(key, searchKeyInFile(mapped, key, channel.size())); } catch (IOException e) { @@ -130,24 +131,27 @@ private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) } private long alignmentBy(long data, long by) { - return data % by == 0 ? data : data + (by - data % by); + return data % by == 0L ? data : data + (by - data % by); } @Override public void close() throws IOException { if (ssTablesDir == null) { + mp.clear(); return; } Set options = Set.of(WRITE, READ, CREATE); - try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(ssTableName), options)) { - long currOffset = 0L; - long allDataLen = 0L; + try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(SS_TABLE_NAME), options)) { + long dataLenght = 0L; + for (var kv : mp.values()) { - allDataLen += alignmentBy(kv.key().byteSize() + kv.value().byteSize() + 16, 8); + dataLenght += alignmentBy(kv.key().byteSize() + kv.value().byteSize() + 16, 8); } + + long currOffset = 0L; MemorySegment mappedSegment = channel.map( - FileChannel.MapMode.READ_WRITE, currOffset, allDataLen, Arena.ofAuto()); + FileChannel.MapMode.READ_WRITE, currOffset, dataLenght, Arena.ofAuto()); for (var kv : mp.values()) { currOffset = dumpSegmentSize(mappedSegment, kv.key().byteSize(), currOffset); From 8474dea5ad4ddc4f5761cfc4004a6ba2fee59924 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 3 Oct 2023 20:18:14 +0300 Subject: [PATCH 10/57] some clean --- src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index edd262863..d251abef5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -18,9 +18,9 @@ import java.util.concurrent.ConcurrentSkipListMap; import static java.lang.Long.min; +import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.WRITE; -import static java.nio.file.StandardOpenOption.CREATE; public class InMemoryDao implements Dao> { From 907b17ac714701708230a666da99ff302b1f26ff Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Tue, 3 Oct 2023 20:28:04 +0300 Subject: [PATCH 11/57] replace all magical values to constants --- .../java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index d251abef5..011f50d18 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -62,7 +62,7 @@ private MemorySegment searchKeyInFile(MemorySegment mapped, MemorySegment key, l long valueByteSize; while (offset < maxSize) { - offset = alignmentBy(offset, 8); + offset = alignmentBy(offset, Long.BYTES); keyByteSize = mapped.get(ValueLayout.JAVA_LONG, offset); offset += Long.BYTES; valueByteSize = mapped.get(ValueLayout.JAVA_LONG, offset); @@ -120,7 +120,7 @@ public int compare(MemorySegment segment1, MemorySegment segment2) { } private long dumpSegmentSize(MemorySegment mapped, long size, long offset) { - long formattedOffset = alignmentBy(offset, 8); + long formattedOffset = alignmentBy(offset, Long.BYTES); mapped.set(ValueLayout.JAVA_LONG, formattedOffset, size); return formattedOffset + Long.BYTES; } @@ -146,7 +146,9 @@ public void close() throws IOException { long dataLenght = 0L; for (var kv : mp.values()) { - dataLenght += alignmentBy(kv.key().byteSize() + kv.value().byteSize() + 16, 8); + dataLenght += alignmentBy( + kv.key().byteSize() + kv.value().byteSize() + 2 * Long.BYTES, + Long.BYTES); } long currOffset = 0L; From 52d32e517e2118cb55b87da8d3b370e0f8b3282b Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 9 Oct 2023 13:51:05 +0300 Subject: [PATCH 12/57] prepare for binsearch --- .../itmo/test/kachmareugene/DaoFactory.java | 2 +- .../itmo/test/kachmareugene/InMemoryDao.java | 93 ++++++++++++------- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index cc9f17d8e..7d9ab8c01 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -36,6 +36,6 @@ public Dao> createDao() { @Override public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config.basePath()); + return new InMemoryDao(config); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 011f50d18..20c0416b4 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -1,6 +1,7 @@ package ru.vk.itmo.test.kachmareugene; import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; @@ -9,13 +10,12 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.util.Comparator; -import java.util.Iterator; -import java.util.Set; -import java.util.SortedMap; +import java.util.*; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.stream.Stream; import static java.lang.Long.min; import static java.nio.file.StandardOpenOption.CREATE; @@ -27,16 +27,36 @@ public class InMemoryDao implements Dao> { private final Comparator memorySegmentComparatorImpl = new MemorySegmentComparator(); private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); - private final Path ssTablesDir; - private static final String SS_TABLE_NAME = "ssTable.txt"; - - public InMemoryDao() { - this.ssTablesDir = null; + private final List ssTables = new ArrayList<>(); + private final List ssTablesIndexes = new ArrayList<>(); + private static final String SS_TABLE_COMMON_PREF = "ssTable"; + + // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen + private static final String INDEX_COMMON_PREF = "index"; + private final Arena arena; + + private void openFiles(Path dir, String fileNamePref, List storage) { + try (Stream tabels = Files.find(dir, 1, + (path, ignore) -> path.getFileName().startsWith(fileNamePref))) { + tabels.forEach(t -> { + try { + storage.add(FileChannel.open(t, READ)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(String.format("Cannot open %s %s", fileNamePref, e.getMessage())); + } } - public InMemoryDao(Path path) { - this.ssTablesDir = path; + public InMemoryDao(Config conf) { + this.arena = Arena.ofShared(); + this.ssTablesDir = conf.basePath(); + + openFiles(conf.basePath(), SS_TABLE_COMMON_PREF, ssTables); + openFiles(conf.basePath(), INDEX_COMMON_PREF, ssTablesIndexes); } @Override @@ -56,6 +76,7 @@ public Iterator> get(MemorySegment from, MemorySegment to) return dataSlice.values().iterator(); } + // fixme give offset in file private MemorySegment searchKeyInFile(MemorySegment mapped, MemorySegment key, long maxSize) { long offset = 0L; long keyByteSize; @@ -86,7 +107,7 @@ public Entry get(MemorySegment key) { return null; } - try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(SS_TABLE_NAME), READ)) { + try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF), READ)) { MemorySegment mapped = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), Arena.ofAuto()); return new BaseEntry<>(key, searchKeyInFile(mapped, key, channel.size())); } catch (IOException e) { @@ -119,10 +140,9 @@ public int compare(MemorySegment segment1, MemorySegment segment2) { } } - private long dumpSegmentSize(MemorySegment mapped, long size, long offset) { - long formattedOffset = alignmentBy(offset, Long.BYTES); - mapped.set(ValueLayout.JAVA_LONG, formattedOffset, size); - return formattedOffset + Long.BYTES; + private long dumpLong(MemorySegment mapped, long value, long offset) { + mapped.set(ValueLayout.JAVA_LONG, offset, value); + return offset + Long.BYTES; } private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { @@ -130,10 +150,6 @@ private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) return offset + data.byteSize(); } - private long alignmentBy(long data, long by) { - return data % by == 0L ? data : data + (by - data % by); - } - @Override public void close() throws IOException { if (ssTablesDir == null) { @@ -142,24 +158,37 @@ public void close() throws IOException { } Set options = Set.of(WRITE, READ, CREATE); - try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(SS_TABLE_NAME), options)) { - long dataLenght = 0L; + + try (FileChannel ssTableChannel = + FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + ssTables.size()), options); + FileChannel indexChannel = + FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + ssTables.size()), options)) { + + long ssTableLenght = 0L; + long indexLength = mp.size() * (long) Long.BYTES * 3; for (var kv : mp.values()) { - dataLenght += alignmentBy( - kv.key().byteSize() + kv.value().byteSize() + 2 * Long.BYTES, - Long.BYTES); + ssTableLenght += + kv.key().byteSize() + kv.value().byteSize(); } - long currOffset = 0L; - MemorySegment mappedSegment = channel.map( - FileChannel.MapMode.READ_WRITE, currOffset, dataLenght, Arena.ofAuto()); + long currOffsetSSTable = 0L; + long currOffsetIndex = 0L; + + MemorySegment mappedSSTable = ssTableChannel.map( + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, Arena.ofConfined()); + + MemorySegment mappedIndex = ssTableChannel.map( + FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, Arena.ofConfined()); for (var kv : mp.values()) { - currOffset = dumpSegmentSize(mappedSegment, kv.key().byteSize(), currOffset); - currOffset = dumpSegmentSize(mappedSegment, kv.value().byteSize(), currOffset); - currOffset = dumpSegment(mappedSegment, kv.key(), currOffset); - currOffset = dumpSegment(mappedSegment, kv.value(), currOffset); + currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); + currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + + currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = dumpSegment(mappedSSTable, kv.value(), currOffsetSSTable); + currOffsetIndex = dumpLong(mappedIndex, kv.value().byteSize(), currOffsetIndex); } } finally { mp.clear(); From 166cb3dd83cde35351850aab0e056ac1df80e744 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 9 Oct 2023 13:52:40 +0300 Subject: [PATCH 13/57] fixed chenges --- src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 8b65fa9bd..b86130bea 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -20,7 +20,7 @@ public MemorySegment fromString(String data) { if (data == null) { return null; } - return Arena.ofAuto().allocateArray(ValueLayout.JAVA_BYTE, data.getBytes(StandardCharsets.UTF_8)); + return MemorySegment.ofArray(data.getBytes(StandardCharsets.UTF_8)); } @Override From 3f6eb178cb7be6b3947f73f9796dd0d5d4d238ec Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 9 Oct 2023 23:02:19 +0300 Subject: [PATCH 14/57] changes --- src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index b86130bea..cf64195dd 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -3,7 +3,6 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; From 6468b3ed75885d3a8997464ea40ac534960b66e5 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 16 Oct 2023 22:19:48 +0300 Subject: [PATCH 15/57] 3rd task init --- .../itmo/test/kachmareugene/InMemoryDao.java | 149 +----------- .../MemorySegmentComparator.java | 26 +++ .../test/kachmareugene/SSTableIterator.java | 109 +++++++++ .../test/kachmareugene/SSTableRowInfo.java | 17 ++ .../kachmareugene/SSTablesController.java | 221 ++++++++++++++++++ 5 files changed, 381 insertions(+), 141 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/MemorySegmentComparator.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 20c0416b4..2d2860278 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -1,62 +1,27 @@ package ru.vk.itmo.test.kachmareugene; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; import java.io.IOException; -import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.OpenOption; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.stream.Stream; - -import static java.lang.Long.min; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.WRITE; public class InMemoryDao implements Dao> { private final Comparator memorySegmentComparatorImpl = new MemorySegmentComparator(); private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); - private final Path ssTablesDir; - private final List ssTables = new ArrayList<>(); - private final List ssTablesIndexes = new ArrayList<>(); - private static final String SS_TABLE_COMMON_PREF = "ssTable"; - // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen - private static final String INDEX_COMMON_PREF = "index"; - private final Arena arena; + private final SSTablesController controller; - private void openFiles(Path dir, String fileNamePref, List storage) { - try (Stream tabels = Files.find(dir, 1, - (path, ignore) -> path.getFileName().startsWith(fileNamePref))) { - tabels.forEach(t -> { - try { - storage.add(FileChannel.open(t, READ)); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } catch (IOException e) { - throw new RuntimeException(String.format("Cannot open %s %s", fileNamePref, e.getMessage())); - } + public InMemoryDao() { + this.controller = new SSTablesController(memorySegmentComparatorImpl); } - public InMemoryDao(Config conf) { - this.arena = Arena.ofShared(); - this.ssTablesDir = conf.basePath(); - - openFiles(conf.basePath(), SS_TABLE_COMMON_PREF, ssTables); - openFiles(conf.basePath(), INDEX_COMMON_PREF, ssTablesIndexes); + this.controller = new SSTablesController(conf.basePath(), memorySegmentComparatorImpl); } @Override @@ -73,28 +38,7 @@ public Iterator> get(MemorySegment from, MemorySegment to) dataSlice = mp.subMap(from, to); } - return dataSlice.values().iterator(); - } - - // fixme give offset in file - private MemorySegment searchKeyInFile(MemorySegment mapped, MemorySegment key, long maxSize) { - long offset = 0L; - long keyByteSize; - long valueByteSize; - - while (offset < maxSize) { - offset = alignmentBy(offset, Long.BYTES); - keyByteSize = mapped.get(ValueLayout.JAVA_LONG, offset); - offset += Long.BYTES; - valueByteSize = mapped.get(ValueLayout.JAVA_LONG, offset); - offset += Long.BYTES; - - if (memorySegmentComparatorImpl.compare(key, mapped.asSlice(offset, keyByteSize)) == 0) { - return mapped.asSlice(offset + keyByteSize, valueByteSize); - } - offset += keyByteSize + valueByteSize; - } - return null; + return new SSTableIterator(dataSlice.values().iterator(), controller, from, to); } @Override @@ -103,16 +47,7 @@ public Entry get(MemorySegment key) { return mp.get(key); } - if (ssTablesDir == null) { - return null; - } - - try (FileChannel channel = FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF), READ)) { - MemorySegment mapped = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), Arena.ofAuto()); - return new BaseEntry<>(key, searchKeyInFile(mapped, key, channel.size())); - } catch (IOException e) { - return null; - } + return controller.getRow(controller.searchInSStables(key)); } @Override @@ -120,78 +55,10 @@ public void upsert(Entry entry) { mp.put(entry.key(), entry); } - private static class MemorySegmentComparator implements Comparator { - - @Override - public int compare(MemorySegment segment1, MemorySegment segment2) { - - long firstDiffByte = segment1.mismatch(segment2); - - if (firstDiffByte == -1) { - return 0; - } - - if (firstDiffByte == min(segment1.byteSize(), segment2.byteSize())) { - return firstDiffByte == segment1.byteSize() ? -1 : 1; - } - - return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, firstDiffByte), - segment2.get(ValueLayout.JAVA_BYTE, firstDiffByte)); - } - } - - private long dumpLong(MemorySegment mapped, long value, long offset) { - mapped.set(ValueLayout.JAVA_LONG, offset, value); - return offset + Long.BYTES; - } - - private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { - MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); - return offset + data.byteSize(); - } @Override public void close() throws IOException { - if (ssTablesDir == null) { - mp.clear(); - return; - } - - Set options = Set.of(WRITE, READ, CREATE); - - try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + ssTables.size()), options); - FileChannel indexChannel = - FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + ssTables.size()), options)) { - - long ssTableLenght = 0L; - long indexLength = mp.size() * (long) Long.BYTES * 3; - - for (var kv : mp.values()) { - ssTableLenght += - kv.key().byteSize() + kv.value().byteSize(); - } - - long currOffsetSSTable = 0L; - long currOffsetIndex = 0L; - - MemorySegment mappedSSTable = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, Arena.ofConfined()); - - MemorySegment mappedIndex = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, Arena.ofConfined()); - - for (var kv : mp.values()) { - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); - - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, kv.value(), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, kv.value().byteSize(), currOffsetIndex); - } - } finally { - mp.clear(); - } + controller.dumpMemTableToSStable(mp); + mp.clear(); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/MemorySegmentComparator.java new file mode 100644 index 000000000..f502a8d86 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/MemorySegmentComparator.java @@ -0,0 +1,26 @@ +package ru.vk.itmo.test.kachmareugene; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +import static java.lang.Long.min; + +public class MemorySegmentComparator implements Comparator { + @Override + public int compare(MemorySegment segment1, MemorySegment segment2) { + + long firstDiffByte = segment1.mismatch(segment2); + + if (firstDiffByte == -1) { + return 0; + } + + if (firstDiffByte == min(segment1.byteSize(), segment2.byteSize())) { + return firstDiffByte == segment1.byteSize() ? -1 : 1; + } + + return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, firstDiffByte), + segment2.get(ValueLayout.JAVA_BYTE, firstDiffByte)); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java new file mode 100644 index 000000000..0216529e1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -0,0 +1,109 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.*; + +public class SSTableIterator implements Iterator> { + + private final Iterator> memTableIterator; + private final SSTablesController controller; + private final Comparator comp = new MemorySegmentComparator(); + private final SortedMap mp = new TreeMap<>(comp); + private final MemorySegment from, to; + private Entry head = null; + + + public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { + memTableIterator = it; + this.controller = controller; + + this.from = from; + this.to = to; + + positioningIterator(); + + } + + private void insertNew(SSTableRowInfo info) { + Entry kv = controller.getRow(info); + + if (kv == null) { + return; + } + + if (!mp.containsKey(kv.key())) { + mp.put(kv.key(), info); + return; + } + SSTableRowInfo old = mp.get(kv.key()); + + SSTableRowInfo oldInfo = old.SSTableInd > info.SSTableInd ? old : info; + SSTableRowInfo newInfo = old.SSTableInd < info.SSTableInd ? info : old; + + mp.put(controller.getRow(newInfo).key(), newInfo); + + // tail recursion + insertNew(controller.getNextInfo(oldInfo, to)); + } + + private void positioningIterator() { + List rawData = controller.firstGreaterKeys(from); + + for (var info : rawData) { + insertNew(info); + } + } + + @Override + public boolean hasNext() { + return !mp.isEmpty() || hasHead(); + } + + private Entry getHead() { + if (head == null) { + head = memTableIterator.next(); + } + return head; + } + private Entry moveAndGet() { + getHead(); + var ans = head; + head = null; + return ans; + } + private boolean hasHead() { + return head != null || memTableIterator.hasNext(); + } + + @Override + public Entry next() { + if (mp.isEmpty()) { + return moveAndGet(); + } + + Map.Entry minSStablesEntry = mp.firstEntry(); + + if (!hasHead()) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + return controller.getRow(minSStablesEntry.getValue()); + } + + + + if (comp.compare(getHead().key(), minSStablesEntry.getKey()) < 0) { + return moveAndGet(); + } + + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + + if (comp.compare(getHead().key(), minSStablesEntry.getKey()) > 0) { + return controller.getRow(minSStablesEntry.getValue()); + } + + return moveAndGet(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java new file mode 100644 index 000000000..16afe77c4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -0,0 +1,17 @@ +package ru.vk.itmo.test.kachmareugene; + +public class SSTableRowInfo { + long keyOffset; + long valueOffset; + long keySize; + long valueSize; + int SSTableInd; + + public SSTableRowInfo(long keyOffset, long valueOffset, long keySize, long valueSize, int SSTableInd) { + this.keyOffset = keyOffset; + this.valueOffset = valueOffset; + this.keySize = keySize; + this.valueSize = valueSize; + this.SSTableInd = SSTableInd; + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java new file mode 100644 index 000000000..830698fa1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -0,0 +1,221 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Stream; + +import static java.nio.file.StandardOpenOption.*; + +public class SSTablesController { + + private final Path ssTablesDir; + + private final List ssTables = new ArrayList<>(); + private final List ssTablesIndexes = new ArrayList<>(); + private static final String SS_TABLE_COMMON_PREF = "ssTable"; + + // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen + private static final long oneLineSize = 4 * Long.BYTES; + private static final String INDEX_COMMON_PREF = "index"; + private final Arena arenaForReading = Arena.ofShared(); + private final Comparator segComp; + + public SSTablesController(Path dir, Comparator com) { + this.ssTablesDir = dir; + this.segComp = com; + + + openFiles(dir, SS_TABLE_COMMON_PREF, ssTables); + openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes); + } + public SSTablesController(Comparator com) { + this.ssTablesDir = null; + this.segComp = com; + } + + private void openFiles(Path dir, String fileNamePref, List storage) { + try (Stream tabels = Files.find(dir, 1, + (path, ignore) -> path.getFileName().toString().startsWith(fileNamePref))) { + tabels.forEach(t -> { + try (FileChannel channel = FileChannel.open(t, READ)) { + storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(String.format("Cannot open %s %s", fileNamePref, e.getMessage())); + } + } + + private boolean lessThenKey(MemorySegment mappedIndex, long lineInBytesOffset, MemorySegment mappedData, MemorySegment key) { + long offset = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset); + long size = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset + Long.BYTES); + + return segComp.compare(mappedData.asSlice(offset, size), key) < 0; + } + + /** + Gives offset for line in index file + **/ + private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { + long l = -1; + long r = mappedIndex.byteSize() / oneLineSize; + + while (r - l > 1) { + long mid = (l + r) / 2; + + if (lessThenKey(mappedIndex, mid * oneLineSize, mappedData, key) ) { + l = mid; + } else { + r = mid; + } + } + return r == (mappedIndex.byteSize() / oneLineSize) ? -1 : r; + } + + /** + * - + * @return List ordered form the latest created sstable to the first. + */ + public List firstGreaterKeys(MemorySegment key) { + List ans = new ArrayList<>(); + + for (int i = ssTables.size() - 1; i >= 0; i--) { + long entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + if (entryIndexesLine < 0) { + continue; + } + ans.add(createRowInfo(i, entryIndexesLine)); + } + return ans; + } + // fixme for loop and smart set + private SSTableRowInfo createRowInfo(int ind, long offset) { + long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, offset * oneLineSize); + long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, offset * oneLineSize + Long.BYTES); + + long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,offset * oneLineSize + Long.BYTES * 2); + long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,offset * oneLineSize + Long.BYTES * 3); + return new SSTableRowInfo(start, size, start1, size1, ind); + } + + public SSTableRowInfo searchInSStables(MemorySegment key) { + for (int i = ssTablesIndexes.size() - 1; i >= 0; i--) { + long ind = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + if (ind < 0) { + continue; + } + + return createRowInfo(i, ind); + } + return null; + } + + public Entry getRow(SSTableRowInfo info) { + if (info == null || info.valueSize == 0) { + return null; + } + + var key = ssTables.get(info.SSTableInd).asSlice(info.keyOffset, info.keySize); + var value = ssTables.get(info.SSTableInd).asSlice(info.valueOffset, info.valueSize); + + return new BaseEntry<>(key, value); + } + + /** + * Ignores deleted values. + */ + public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { + + for (long t = info.keyOffset / oneLineSize + 1; t < ssTablesIndexes.size() / oneLineSize; t++) { + var inf = createRowInfo(info.SSTableInd, t); + Entry row = getRow(inf); + + if (row.value().byteSize() == 0) { + continue; + } + + if (segComp.compare(row.key(), maxKey) < 0) { + return inf; + } + } + + + return null; + } + + private long dumpLong(MemorySegment mapped, long value, long offset) { + mapped.set(ValueLayout.JAVA_LONG, offset, value); + return offset + Long.BYTES; + } + + private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { + MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); + return offset + data.byteSize(); + } + + + public void dumpMemTableToSStable(SortedMap> mp) throws IOException { + + // fixme as nullpointer exeption + if (ssTablesDir == null) { + arenaForReading.close(); + return; + } + + Set options = Set.of(WRITE, READ, CREATE); + + try (FileChannel ssTableChannel = + FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + ssTables.size()), options); + FileChannel indexChannel = + FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + ssTables.size()), options)) { + + long ssTableLenght = 0L; + long indexLength = mp.size() * oneLineSize; + + for (var kv : mp.values()) { + ssTableLenght += + kv.key().byteSize() + getValueOrNull(kv).byteSize(); + } + + long currOffsetSSTable = 0L; + long currOffsetIndex = 0L; + + MemorySegment mappedSSTable = ssTableChannel.map( + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, Arena.ofConfined()); + + MemorySegment mappedIndex = indexChannel.map( + FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, Arena.ofConfined()); + + for (var kv : mp.values()) { + currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); + currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + + currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); + currOffsetIndex = dumpLong(mappedIndex, getValueOrNull(kv).byteSize(), currOffsetIndex); + } + } finally { + arenaForReading.close(); + } + } + private MemorySegment getValueOrNull(Entry kv) { + MemorySegment value = kv.value(); + if (kv.value() == null) { + value = MemorySegment.NULL; + } + return value; + } +} From 97a00f3eee5888feee72c44373070b3c0dfb453b Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 16:39:30 +0300 Subject: [PATCH 16/57] add 3rd task --- .../itmo/test/kachmareugene/DaoFactory.java | 9 +- .../itmo/test/kachmareugene/InMemoryDao.java | 13 ++- .../kachmareugene/MemSegComparatorNull.java | 17 ++++ .../test/kachmareugene/SSTableIterator.java | 98 +++++++++++++------ .../test/kachmareugene/SSTableRowInfo.java | 18 +++- .../kachmareugene/SSTablesController.java | 66 ++++++++----- 6 files changed, 160 insertions(+), 61 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 7d9ab8c01..0a37be5ed 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -9,11 +9,14 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@ru.vk.itmo.test.DaoFactory(stage = 2) +@ru.vk.itmo.test.DaoFactory(stage = 3) public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { - return new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); + if (memorySegment == null) { + return null; + } + return new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); } @Override @@ -38,4 +41,4 @@ public Dao> createDao() { public Dao> createDao(Config config) throws IOException { return new InMemoryDao(config); } -} +} \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 2d2860278..b9bec18e7 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -18,10 +18,10 @@ public class InMemoryDao implements Dao> { private final SSTablesController controller; public InMemoryDao() { - this.controller = new SSTablesController(memorySegmentComparatorImpl); + this.controller = new SSTablesController(new MemSegComparatorNull()); } public InMemoryDao(Config conf) { - this.controller = new SSTablesController(conf.basePath(), memorySegmentComparatorImpl); + this.controller = new SSTablesController(conf.basePath(), new MemSegComparatorNull()); } @Override @@ -44,10 +44,13 @@ public Iterator> get(MemorySegment from, MemorySegment to) @Override public Entry get(MemorySegment key) { if (mp.containsKey(key)) { + if (mp.get(key).value() == null) { + return null; + } return mp.get(key); } - - return controller.getRow(controller.searchInSStables(key)); + var res = controller.getRow(controller.searchInSStables(key)); + return res.value() == null ? null : res; } @Override @@ -61,4 +64,4 @@ public void close() throws IOException { controller.dumpMemTableToSStable(mp); mp.clear(); } -} +} \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java new file mode 100644 index 000000000..eaff57f3c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java @@ -0,0 +1,17 @@ +package ru.vk.itmo.test.kachmareugene; + +import java.lang.foreign.MemorySegment; + +public class MemSegComparatorNull extends MemorySegmentComparator { + @Override + public int compare(MemorySegment segment1, MemorySegment segment2) { + if (segment1 == null) { + return -1; + } + if (segment2 == null) { + return -1; + } + + return super.compare(segment1, segment2); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 0216529e1..ed3c3cdd5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -9,11 +9,11 @@ public class SSTableIterator implements Iterator> { private final Iterator> memTableIterator; private final SSTablesController controller; - private final Comparator comp = new MemorySegmentComparator(); + private final Comparator comp = new MemSegComparatorNull(); private final SortedMap mp = new TreeMap<>(comp); private final MemorySegment from, to; private Entry head = null; - + private Entry keeper = null; public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { memTableIterator = it; @@ -25,7 +25,6 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } - private void insertNew(SSTableRowInfo info) { Entry kv = controller.getRow(info); @@ -39,7 +38,7 @@ private void insertNew(SSTableRowInfo info) { } SSTableRowInfo old = mp.get(kv.key()); - SSTableRowInfo oldInfo = old.SSTableInd > info.SSTableInd ? old : info; + SSTableRowInfo oldInfo = old.SSTableInd > info.SSTableInd ? info: old ; SSTableRowInfo newInfo = old.SSTableInd < info.SSTableInd ? info : old; mp.put(controller.getRow(newInfo).key(), newInfo); @@ -58,52 +57,95 @@ private void positioningIterator() { @Override public boolean hasNext() { - return !mp.isEmpty() || hasHead(); + if (keeper == null) { + changeState(); + } + return keeper != null; } private Entry getHead() { if (head == null) { - head = memTableIterator.next(); + if (memTableIterator.hasNext()) { + head = memTableIterator.next(); + if (comp.compare(head.key(), to) >= 0) { + head = null; + } + } } return head; } private Entry moveAndGet() { - getHead(); var ans = head; - head = null; + do { + getHead(); + ans = head; + head = null; + } while (ans != null && ans.value() == null); return ans; } - private boolean hasHead() { - return head != null || memTableIterator.hasNext(); - } - @Override - public Entry next() { - if (mp.isEmpty()) { - return moveAndGet(); - } + private void changeState() { + while (true) { + Map.Entry minSStablesEntry = getFirstMin(); - Map.Entry minSStablesEntry = mp.firstEntry(); + if (minSStablesEntry == null) { + keeper = moveAndGet(); + return; + } + + if (getHead() == null) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + keeper = controller.getRow(minSStablesEntry.getValue()); + return; + } + + int res = comp.compare(getHead().key(), minSStablesEntry.getKey()); + + if (res < 0) { + keeper = moveAndGet(); + return; + } + + if (res > 0) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + keeper = controller.getRow(minSStablesEntry.getValue()); + return; + } - if (!hasHead()) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - return controller.getRow(minSStablesEntry.getValue()); + keeper = moveAndGet(); + return; } + } + private boolean isBetween(MemorySegment who) { + int res1 = comp.compare(from, who); + int res2 = comp.compare(who, to); + return res1 <= 0 && res2 < 0; + } + private Map.Entry getFirstMin() { + Map.Entry minSStablesEntry = mp.firstEntry(); - if (comp.compare(getHead().key(), minSStablesEntry.getKey()) < 0) { - return moveAndGet(); - } - - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + while (!mp.isEmpty() && (minSStablesEntry.getValue().isDeletedData() || !isBetween(minSStablesEntry.getKey()))) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - if (comp.compare(getHead().key(), minSStablesEntry.getKey()) > 0) { - return controller.getRow(minSStablesEntry.getValue()); + minSStablesEntry = mp.firstEntry(); } + return minSStablesEntry; + } - return moveAndGet(); + @Override + public Entry next() { + if (keeper == null) { + changeState(); + } + var ans = keeper; + keeper = null; + return ans; } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 16afe77c4..2be372c2c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -4,14 +4,28 @@ public class SSTableRowInfo { long keyOffset; long valueOffset; long keySize; - long valueSize; + long rowShift; + private long valueSize; int SSTableInd; - public SSTableRowInfo(long keyOffset, long valueOffset, long keySize, long valueSize, int SSTableInd) { + public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int SSTableInd, long rowShift) { this.keyOffset = keyOffset; this.valueOffset = valueOffset; this.keySize = keySize; this.valueSize = valueSize; this.SSTableInd = SSTableInd; + this.rowShift = rowShift; + } + + public boolean isDeletedData() { + return valueSize < 0; + } + + public long getValueSize() { + return valueSize; + } + + public long totalShift() { + return keyOffset + keySize + valueSize; } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 830698fa1..da5e5563f 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -11,6 +11,12 @@ import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Stream; @@ -20,6 +26,8 @@ public class SSTablesController { private final Path ssTablesDir; + private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); + private final List ssTables = new ArrayList<>(); private final List ssTablesIndexes = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; @@ -58,11 +66,11 @@ private void openFiles(Path dir, String fileNamePref, List storag } } - private boolean lessThenKey(MemorySegment mappedIndex, long lineInBytesOffset, MemorySegment mappedData, MemorySegment key) { + private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, MemorySegment mappedData, MemorySegment key) { long offset = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset); long size = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset + Long.BYTES); - return segComp.compare(mappedData.asSlice(offset, size), key) < 0; + return segComp.compare(key, mappedData.asSlice(offset, size)) > 0; } /** @@ -75,7 +83,9 @@ private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedDat while (r - l > 1) { long mid = (l + r) / 2; - if (lessThenKey(mappedIndex, mid * oneLineSize, mappedData, key) ) { + + // key >= m + if (greaterThen(mappedIndex, mid * oneLineSize, mappedData, key) ) { l = mid; } else { r = mid; @@ -101,13 +111,13 @@ public List firstGreaterKeys(MemorySegment key) { return ans; } // fixme for loop and smart set - private SSTableRowInfo createRowInfo(int ind, long offset) { - long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, offset * oneLineSize); - long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, offset * oneLineSize + Long.BYTES); + private SSTableRowInfo createRowInfo(int ind, long rowShift) { + long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * oneLineSize); + long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * oneLineSize + Long.BYTES); - long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,offset * oneLineSize + Long.BYTES * 2); - long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,offset * oneLineSize + Long.BYTES * 3); - return new SSTableRowInfo(start, size, start1, size1, ind); + long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * oneLineSize + Long.BYTES * 2); + long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * oneLineSize + Long.BYTES * 3); + return new SSTableRowInfo(start, size, start1, size1, ind, rowShift); } public SSTableRowInfo searchInSStables(MemorySegment key) { @@ -123,12 +133,16 @@ public SSTableRowInfo searchInSStables(MemorySegment key) { } public Entry getRow(SSTableRowInfo info) { - if (info == null || info.valueSize == 0) { + if (info == null) { return null; } var key = ssTables.get(info.SSTableInd).asSlice(info.keyOffset, info.keySize); - var value = ssTables.get(info.SSTableInd).asSlice(info.valueOffset, info.valueSize); + + if (info.isDeletedData()) { + return new BaseEntry<>(key, null); + } + var value = ssTables.get(info.SSTableInd).asSlice(info.valueOffset, info.getValueSize()); return new BaseEntry<>(key, value); } @@ -137,15 +151,15 @@ public Entry getRow(SSTableRowInfo info) { * Ignores deleted values. */ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { - - for (long t = info.keyOffset / oneLineSize + 1; t < ssTablesIndexes.size() / oneLineSize; t++) { + // fixme t0?? + for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.SSTableInd).byteSize() / oneLineSize; t++) { var inf = createRowInfo(info.SSTableInd, t); - Entry row = getRow(inf); - if (row.value().byteSize() == 0) { + if (inf.isDeletedData()) { continue; } + Entry row = getRow(inf); if (segComp.compare(row.key(), maxKey) < 0) { return inf; } @@ -168,18 +182,17 @@ private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) public void dumpMemTableToSStable(SortedMap> mp) throws IOException { - // fixme as nullpointer exeption if (ssTablesDir == null) { arenaForReading.close(); return; } - + LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); Set options = Set.of(WRITE, READ, CREATE); - try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + ssTables.size()), options); + FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), options); FileChannel indexChannel = - FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + ssTables.size()), options)) { + FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), options); + Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; long indexLength = mp.size() * oneLineSize; @@ -193,10 +206,10 @@ public void dumpMemTableToSStable(SortedMap> long currOffsetIndex = 0L; MemorySegment mappedSSTable = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, Arena.ofConfined()); + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); MemorySegment mappedIndex = indexChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, Arena.ofConfined()); + FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); for (var kv : mp.values()) { currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); @@ -205,12 +218,19 @@ public void dumpMemTableToSStable(SortedMap> currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, getValueOrNull(kv).byteSize(), currOffsetIndex); + currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); } } finally { arenaForReading.close(); } } + + private long rightByteSize(Entry memSeg) { + if (memSeg.value() == null) { + return -1; + } + return memSeg.value().byteSize(); + } private MemorySegment getValueOrNull(Entry kv) { MemorySegment value = kv.value(); if (kv.value() == null) { From aaac68daac37b9f3ac944f71c406ffabe3fa49dd Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 16:51:09 +0300 Subject: [PATCH 17/57] add 3rd task --- src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index b9bec18e7..2a23ededf 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -58,7 +58,6 @@ public void upsert(Entry entry) { mp.put(entry.key(), entry); } - @Override public void close() throws IOException { controller.dumpMemTableToSStable(mp); From a4c2e7db2a854b725b510c9daef489539281f2cd Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 18:21:57 +0300 Subject: [PATCH 18/57] add 3rd task --- .../itmo/test/kachmareugene/InMemoryDao.java | 5 +- .../test/kachmareugene/SSTableIterator.java | 32 ++++++----- .../test/kachmareugene/SSTableRowInfo.java | 9 +-- .../kachmareugene/SSTablesController.java | 56 +++++++++---------- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 2a23ededf..f2ebed1a5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -6,7 +6,9 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.util.*; +import java.util.Comparator; +import java.util.Iterator; +import java.util.SortedMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDao implements Dao> { @@ -20,6 +22,7 @@ public class InMemoryDao implements Dao> { public InMemoryDao() { this.controller = new SSTablesController(new MemSegComparatorNull()); } + public InMemoryDao(Config conf) { this.controller = new SSTablesController(conf.basePath(), new MemSegComparatorNull()); } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index ed3c3cdd5..5ba363d18 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -3,7 +3,7 @@ import ru.vk.itmo.Entry; import java.lang.foreign.MemorySegment; -import java.util.*; +import java.util.*; public class SSTableIterator implements Iterator> { @@ -11,11 +11,13 @@ public class SSTableIterator implements Iterator> { private final SSTablesController controller; private final Comparator comp = new MemSegComparatorNull(); private final SortedMap mp = new TreeMap<>(comp); - private final MemorySegment from, to; - private Entry head = null; - private Entry keeper = null; + private final MemorySegment from; + private final MemorySegment to; + private Entry head; + private Entry keeper; - public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { + public SSTableIterator(Iterator> it, SSTablesController controller, + MemorySegment from, MemorySegment to) { memTableIterator = it; this.controller = controller; @@ -25,6 +27,7 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } + private void insertNew(SSTableRowInfo info) { Entry kv = controller.getRow(info); @@ -38,8 +41,8 @@ private void insertNew(SSTableRowInfo info) { } SSTableRowInfo old = mp.get(kv.key()); - SSTableRowInfo oldInfo = old.SSTableInd > info.SSTableInd ? info: old ; - SSTableRowInfo newInfo = old.SSTableInd < info.SSTableInd ? info : old; + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old ; + SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; mp.put(controller.getRow(newInfo).key(), newInfo); @@ -64,16 +67,15 @@ public boolean hasNext() { } private Entry getHead() { - if (head == null) { - if (memTableIterator.hasNext()) { - head = memTableIterator.next(); - if (comp.compare(head.key(), to) >= 0) { - head = null; - } + if (head == null && memTableIterator.hasNext()) { + head = memTableIterator.next(); + if (comp.compare(head.key(), to) >= 0) { + head = null; } } return head; } + private Entry moveAndGet() { var ans = head; do { @@ -117,7 +119,6 @@ private void changeState() { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); keeper = moveAndGet(); - return; } } @@ -130,7 +131,8 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); - while (!mp.isEmpty() && (minSStablesEntry.getValue().isDeletedData() || !isBetween(minSStablesEntry.getKey()))) { + while (!mp.isEmpty() && + (minSStablesEntry.getValue().isDeletedData() || !isBetween(minSStablesEntry.getKey()))) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 2be372c2c..309a26400 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -5,15 +5,16 @@ public class SSTableRowInfo { long valueOffset; long keySize; long rowShift; - private long valueSize; - int SSTableInd; + private final long valueSize; + int ssTableInd; - public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int SSTableInd, long rowShift) { + public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, + long valueSize, int ssTableInd, long rowShift) { this.keyOffset = keyOffset; this.valueOffset = valueOffset; this.keySize = keySize; this.valueSize = valueSize; - this.SSTableInd = SSTableInd; + this.ssTableInd = ssTableInd; this.rowShift = rowShift; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index da5e5563f..1eb402b92 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -11,13 +11,14 @@ import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.text.SimpleDateFormat; -import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.LocalTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; import java.util.stream.Stream; import static java.nio.file.StandardOpenOption.*; @@ -33,7 +34,7 @@ public class SSTablesController { private static final String SS_TABLE_COMMON_PREF = "ssTable"; // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen - private static final long oneLineSize = 4 * Long.BYTES; + private static final long ONE_LINE_SIZE = 4 * Long.BYTES; private static final String INDEX_COMMON_PREF = "index"; private final Arena arenaForReading = Arena.ofShared(); private final Comparator segComp; @@ -46,6 +47,7 @@ public SSTablesController(Path dir, Comparator com) { openFiles(dir, SS_TABLE_COMMON_PREF, ssTables); openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes); } + public SSTablesController(Comparator com) { this.ssTablesDir = null; this.segComp = com; @@ -66,38 +68,32 @@ private void openFiles(Path dir, String fileNamePref, List storag } } - private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, MemorySegment mappedData, MemorySegment key) { + private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, + MemorySegment mappedData, MemorySegment key) { long offset = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset); long size = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset + Long.BYTES); return segComp.compare(key, mappedData.asSlice(offset, size)) > 0; } - /** - Gives offset for line in index file - **/ + //Gives offset for line in index file private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { long l = -1; - long r = mappedIndex.byteSize() / oneLineSize; + long r = mappedIndex.byteSize() / ONE_LINE_SIZE; while (r - l > 1) { long mid = (l + r) / 2; - - // key >= m - if (greaterThen(mappedIndex, mid * oneLineSize, mappedData, key) ) { + if (greaterThen(mappedIndex, mid * ONE_LINE_SIZE, mappedData, key)) { l = mid; } else { r = mid; } } - return r == (mappedIndex.byteSize() / oneLineSize) ? -1 : r; + return r == (mappedIndex.byteSize() / ONE_LINE_SIZE) ? -1 : r; } - /** - * - - * @return List ordered form the latest created sstable to the first. - */ + //return - List ordered form the latest created sstable to the first. public List firstGreaterKeys(MemorySegment key) { List ans = new ArrayList<>(); @@ -110,13 +106,13 @@ public List firstGreaterKeys(MemorySegment key) { } return ans; } - // fixme for loop and smart set + private SSTableRowInfo createRowInfo(int ind, long rowShift) { - long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * oneLineSize); - long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * oneLineSize + Long.BYTES); + long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE); + long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE + Long.BYTES); - long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * oneLineSize + Long.BYTES * 2); - long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * oneLineSize + Long.BYTES * 3); + long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 2); + long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 3); return new SSTableRowInfo(start, size, start1, size1, ind, rowShift); } @@ -137,12 +133,12 @@ public Entry getRow(SSTableRowInfo info) { return null; } - var key = ssTables.get(info.SSTableInd).asSlice(info.keyOffset, info.keySize); + var key = ssTables.get(info.ssTableInd).asSlice(info.keyOffset, info.keySize); if (info.isDeletedData()) { return new BaseEntry<>(key, null); } - var value = ssTables.get(info.SSTableInd).asSlice(info.valueOffset, info.getValueSize()); + var value = ssTables.get(info.ssTableInd).asSlice(info.valueOffset, info.getValueSize()); return new BaseEntry<>(key, value); } @@ -151,9 +147,8 @@ public Entry getRow(SSTableRowInfo info) { * Ignores deleted values. */ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { - // fixme t0?? - for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.SSTableInd).byteSize() / oneLineSize; t++) { - var inf = createRowInfo(info.SSTableInd, t); + for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.ssTableInd).byteSize() / ONE_LINE_SIZE; t++) { + var inf = createRowInfo(info.ssTableInd, t); if (inf.isDeletedData()) { continue; @@ -165,7 +160,6 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { } } - return null; } @@ -179,7 +173,6 @@ private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) return offset + data.byteSize(); } - public void dumpMemTableToSStable(SortedMap> mp) throws IOException { if (ssTablesDir == null) { @@ -195,7 +188,7 @@ public void dumpMemTableToSStable(SortedMap> Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; - long indexLength = mp.size() * oneLineSize; + long indexLength = mp.size() * ONE_LINE_SIZE; for (var kv : mp.values()) { ssTableLenght += @@ -231,6 +224,7 @@ private long rightByteSize(Entry memSeg) { } return memSeg.value().byteSize(); } + private MemorySegment getValueOrNull(Entry kv) { MemorySegment value = kv.value(); if (kv.value() == null) { From e35cd9256fe0d1d51196aa14e23216e48d2b10a1 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 18:52:36 +0300 Subject: [PATCH 19/57] add 3rd task --- .../itmo/test/kachmareugene/DaoFactory.java | 2 +- .../itmo/test/kachmareugene/InMemoryDao.java | 2 +- .../test/kachmareugene/SSTableIterator.java | 20 +++++++++++++------ .../kachmareugene/SSTablesController.java | 14 ++++++------- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 0a37be5ed..74efad8e6 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -41,4 +41,4 @@ public Dao> createDao() { public Dao> createDao(Config config) throws IOException { return new InMemoryDao(config); } -} \ No newline at end of file +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index f2ebed1a5..c02f763f5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -66,4 +66,4 @@ public void close() throws IOException { controller.dumpMemTableToSStable(mp); mp.clear(); } -} \ No newline at end of file +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 5ba363d18..f546bffc4 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -1,9 +1,15 @@ package ru.vk.itmo.test.kachmareugene; + import ru.vk.itmo.Entry; import java.lang.foreign.MemorySegment; -import java.util.*; +import java.util.Comparator; +import java.util.Iterator; +import java.util.SortedMap; +import java.util.Map; +import java.util.List; +import java.util.TreeMap; public class SSTableIterator implements Iterator> { @@ -41,7 +47,7 @@ private void insertNew(SSTableRowInfo info) { } SSTableRowInfo old = mp.get(kv.key()); - SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old ; + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; mp.put(controller.getRow(newInfo).key(), newInfo); @@ -95,14 +101,16 @@ private void changeState() { return; } - if (getHead() == null) { + var curHead = getHead(); + + if (curHead == null) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); keeper = controller.getRow(minSStablesEntry.getValue()); return; } - int res = comp.compare(getHead().key(), minSStablesEntry.getKey()); + int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); if (res < 0) { keeper = moveAndGet(); @@ -131,8 +139,8 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); - while (!mp.isEmpty() && - (minSStablesEntry.getValue().isDeletedData() || !isBetween(minSStablesEntry.getKey()))) { + while (!mp.isEmpty() && (minSStablesEntry.getValue().isDeletedData() || + !isBetween(minSStablesEntry.getKey()))) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 1eb402b92..97f9e6b58 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -21,7 +21,9 @@ import java.util.SortedMap; import java.util.stream.Stream; -import static java.nio.file.StandardOpenOption.*; +import static java.nio.file.StandardOpenOption.WRITE; +import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.CREATE; public class SSTablesController { @@ -59,13 +61,9 @@ private void openFiles(Path dir, String fileNamePref, List storag tabels.forEach(t -> { try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); - } catch (IOException e) { - throw new RuntimeException(e); - } + } catch (IOException ignore) {} }); - } catch (IOException e) { - throw new RuntimeException(String.format("Cannot open %s %s", fileNamePref, e.getMessage())); - } + } catch (IOException ignore) {} } private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, @@ -77,7 +75,7 @@ private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, } //Gives offset for line in index file - private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { + private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { long l = -1; long r = mappedIndex.byteSize() / ONE_LINE_SIZE; From ae3723e28f2fda2cb8eef8dc335e4768143e08ff Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 19:05:12 +0300 Subject: [PATCH 20/57] add 3rd task --- .../vk/itmo/test/kachmareugene/SSTableIterator.java | 8 ++++---- .../itmo/test/kachmareugene/SSTablesController.java | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index f546bffc4..27d6c7faf 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -6,9 +6,9 @@ import java.lang.foreign.MemorySegment; import java.util.Comparator; import java.util.Iterator; -import java.util.SortedMap; -import java.util.Map; import java.util.List; +import java.util.Map; +import java.util.SortedMap; import java.util.TreeMap; public class SSTableIterator implements Iterator> { @@ -139,8 +139,8 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); - while (!mp.isEmpty() && (minSStablesEntry.getValue().isDeletedData() || - !isBetween(minSStablesEntry.getKey()))) { + while (!mp.isEmpty() && (minSStablesEntry.getValue().isDeletedData() + || !isBetween(minSStablesEntry.getKey()))) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 97f9e6b58..7fc750a8c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -21,9 +21,9 @@ import java.util.SortedMap; import java.util.stream.Stream; -import static java.nio.file.StandardOpenOption.WRITE; -import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.WRITE; public class SSTablesController { @@ -61,9 +61,13 @@ private void openFiles(Path dir, String fileNamePref, List storag tabels.forEach(t -> { try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); - } catch (IOException ignore) {} + } catch (IOException ignore) { + ; + } }); - } catch (IOException ignore) {} + } catch (IOException ignore) { + ; + } } private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, From fc982a520d4af69283ede36af636536cbd89f9c4 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 19:12:14 +0300 Subject: [PATCH 21/57] add 3rd task --- .../itmo/test/kachmareugene/SSTableIterator.java | 3 +-- .../test/kachmareugene/SSTablesController.java | 15 ++++----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 27d6c7faf..533e39ef8 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -1,6 +1,5 @@ package ru.vk.itmo.test.kachmareugene; - import ru.vk.itmo.Entry; import java.lang.foreign.MemorySegment; @@ -83,7 +82,7 @@ private Entry getHead() { } private Entry moveAndGet() { - var ans = head; + Entry ans; do { getHead(); ans = head; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 7fc750a8c..45a1b2f2a 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -2,7 +2,6 @@ import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; - import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -26,11 +25,8 @@ import static java.nio.file.StandardOpenOption.WRITE; public class SSTablesController { - private final Path ssTablesDir; - - private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); - + private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); private final List ssTables = new ArrayList<>(); private final List ssTablesIndexes = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; @@ -45,7 +41,6 @@ public SSTablesController(Path dir, Comparator com) { this.ssTablesDir = dir; this.segComp = com; - openFiles(dir, SS_TABLE_COMMON_PREF, ssTables); openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes); } @@ -62,7 +57,7 @@ private void openFiles(Path dir, String fileNamePref, List storag try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); } catch (IOException ignore) { - ; + System.err.println();; } }); } catch (IOException ignore) { @@ -121,11 +116,9 @@ private SSTableRowInfo createRowInfo(int ind, long rowShift) { public SSTableRowInfo searchInSStables(MemorySegment key) { for (int i = ssTablesIndexes.size() - 1; i >= 0; i--) { long ind = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); - if (ind < 0) { - continue; + if (ind >= 0) { + return createRowInfo(i, ind); } - - return createRowInfo(i, ind); } return null; } From 504b0b5d88b5f5a509fe907e9e3ac04b07dcb244 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 19:14:44 +0300 Subject: [PATCH 22/57] add 3rd task --- .../vk/itmo/test/kachmareugene/SSTablesController.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 45a1b2f2a..03dfe9119 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -26,7 +26,7 @@ public class SSTablesController { private final Path ssTablesDir; - private final static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); private final List ssTables = new ArrayList<>(); private final List ssTablesIndexes = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; @@ -56,12 +56,12 @@ private void openFiles(Path dir, String fileNamePref, List storag tabels.forEach(t -> { try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); - } catch (IOException ignore) { - System.err.println();; + } catch (IOException e) { + System.err.println(e.getMessage()); } }); - } catch (IOException ignore) { - ; + } catch (IOException e) { + System.err.println(e.getMessage()); } } From 1025d4b897c10905a38576aa1de0883b974151cf Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 19:24:32 +0300 Subject: [PATCH 23/57] clear for code format --- .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 03dfe9119..5781bd7ce 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -3,6 +3,7 @@ import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -57,11 +58,11 @@ private void openFiles(Path dir, String fileNamePref, List storag try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); } catch (IOException e) { - System.err.println(e.getMessage()); + throw new UncheckedIOException(e); } }); } catch (IOException e) { - System.err.println(e.getMessage()); + throw new UncheckedIOException(e); } } From 49f5e614f19631e9d8296ef5b019ec1f92d20131 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 9 Oct 2023 23:02:19 +0300 Subject: [PATCH 24/57] changes --- src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 74efad8e6..9180b9e43 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -4,7 +4,6 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; From 06e42f6c242467a7122738be0938d0670c8b7509 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 20:59:20 +0300 Subject: [PATCH 25/57] fixed --- .../test/kachmareugene/SSTableIterator.java | 51 +++++++++---------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 533e39ef8..d004919c0 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -30,7 +30,6 @@ public SSTableIterator(Iterator> it, SSTablesController con this.to = to; positioningIterator(); - } private void insertNew(SSTableRowInfo info) { @@ -92,41 +91,39 @@ private Entry moveAndGet() { } private void changeState() { - while (true) { - Map.Entry minSStablesEntry = getFirstMin(); - - if (minSStablesEntry == null) { - keeper = moveAndGet(); - return; - } + Map.Entry minSStablesEntry = getFirstMin(); - var curHead = getHead(); + if (minSStablesEntry == null) { + keeper = moveAndGet(); + return; + } - if (curHead == null) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = controller.getRow(minSStablesEntry.getValue()); - return; - } + var curHead = getHead(); - int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); + if (curHead == null) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + keeper = controller.getRow(minSStablesEntry.getValue()); + return; + } - if (res < 0) { - keeper = moveAndGet(); - return; - } + int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); - if (res > 0) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = controller.getRow(minSStablesEntry.getValue()); - return; - } + if (res < 0) { + keeper = moveAndGet(); + return; + } + if (res > 0) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = moveAndGet(); + keeper = controller.getRow(minSStablesEntry.getValue()); + return; } + + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + keeper = moveAndGet(); } private boolean isBetween(MemorySegment who) { From 785bbb557dc43a5f155cce3e33a75d89466f75c7 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 21:02:12 +0300 Subject: [PATCH 26/57] fixed --- src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 9180b9e43..74efad8e6 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -4,6 +4,7 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; From c6ddd98fe1135da4af5962d6d80628bbf22b904b Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 22:00:45 +0300 Subject: [PATCH 27/57] da --- .../java/ru/vk/itmo/test/kachmareugene/SSTablesController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 5781bd7ce..959e99fb5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -181,7 +181,7 @@ public void dumpMemTableToSStable(SortedMap> FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), options); FileChannel indexChannel = FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), options); - Arena saveArena = Arena.ofConfined()) { + Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; long indexLength = mp.size() * ONE_LINE_SIZE; From e6e74d4b5b8572db5ae0e64a942d1304f009d6cb Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 18 Oct 2023 22:34:57 +0300 Subject: [PATCH 28/57] da --- .../java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java | 3 +++ .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index c02f763f5..ab10a5a93 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -53,6 +53,9 @@ public Entry get(MemorySegment key) { return mp.get(key); } var res = controller.getRow(controller.searchInSStables(key)); + if (res == null) { + return null; + } return res.value() == null ? null : res; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 959e99fb5..9b4b0390c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -20,6 +20,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.stream.Stream; +import java.util.Collections; import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; @@ -54,7 +55,9 @@ public SSTablesController(Comparator com) { private void openFiles(Path dir, String fileNamePref, List storage) { try (Stream tabels = Files.find(dir, 1, (path, ignore) -> path.getFileName().toString().startsWith(fileNamePref))) { - tabels.forEach(t -> { + final List list = new ArrayList<>(tabels.toList()); + Collections.sort(list); + list.forEach(t -> { try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); } catch (IOException e) { @@ -171,7 +174,7 @@ private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) public void dumpMemTableToSStable(SortedMap> mp) throws IOException { - if (ssTablesDir == null) { + if (ssTablesDir == null || mp.isEmpty()) { arenaForReading.close(); return; } From 0ec702e9136aa298ae65585d47ae671aa944646f Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 00:00:21 +0300 Subject: [PATCH 29/57] da --- .../test/kachmareugene/SSTableIterator.java | 62 ++++++++++++------- .../kachmareugene/SSTablesController.java | 5 ++ 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index d004919c0..95ba48a35 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -90,40 +90,54 @@ private Entry moveAndGet() { return ans; } + private Entry moveAndGetOnce() { + getHead(); + var ans = head; + head = null; + return ans; + } + private void changeState() { - Map.Entry minSStablesEntry = getFirstMin(); + while (true) { - if (minSStablesEntry == null) { - keeper = moveAndGet(); - return; - } + Map.Entry minSStablesEntry = getFirstMin(); - var curHead = getHead(); + if (minSStablesEntry == null) { + keeper = moveAndGet(); + return; + } - if (curHead == null) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = controller.getRow(minSStablesEntry.getValue()); - return; - } + var curHead = getHead(); - int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); + if (curHead == null) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + keeper = controller.getRow(minSStablesEntry.getValue()); + return; + } - if (res < 0) { - keeper = moveAndGet(); - return; - } + int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); + + if (res < 0) { + keeper = moveAndGet(); + return; + } + + if (res > 0) { + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + keeper = controller.getRow(minSStablesEntry.getValue()); + return; + } - if (res > 0) { + keeper = moveAndGetOnce(); mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = controller.getRow(minSStablesEntry.getValue()); - return; - } - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = moveAndGet(); + if (keeper.value() != null) { + break; + } + } } private boolean isBetween(MemorySegment who) { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 9b4b0390c..d153c446c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -53,6 +53,11 @@ public SSTablesController(Comparator com) { } private void openFiles(Path dir, String fileNamePref, List storage) { + try { + Files.createDirectories(dir); + } catch (IOException e) { + throw new UncheckedIOException(e); + } try (Stream tabels = Files.find(dir, 1, (path, ignore) -> path.getFileName().toString().startsWith(fileNamePref))) { final List list = new ArrayList<>(tabels.toList()); From 2655807092de301b8a2a2de140e811b09a25fec7 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 01:20:08 +0300 Subject: [PATCH 30/57] was late, because one little test:( --- .../test/kachmareugene/SSTableIterator.java | 18 +++++++++--------- .../test/kachmareugene/SSTablesController.java | 4 ---- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 95ba48a35..79d9aec15 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -113,6 +113,9 @@ private void changeState() { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); keeper = controller.getRow(minSStablesEntry.getValue()); + if (keeper != null && keeper.value() == null) { + continue; + } return; } @@ -123,17 +126,15 @@ private void changeState() { return; } + mp.remove(minSStablesEntry.getKey()); + insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + if (res > 0) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); keeper = controller.getRow(minSStablesEntry.getValue()); - return; + } else { + keeper = moveAndGetOnce(); } - keeper = moveAndGetOnce(); - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - if (keeper.value() != null) { break; } @@ -149,8 +150,7 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); - while (!mp.isEmpty() && (minSStablesEntry.getValue().isDeletedData() - || !isBetween(minSStablesEntry.getKey()))) { + while (!mp.isEmpty() && !isBetween(minSStablesEntry.getKey())) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index d153c446c..7a11bbe8f 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -154,10 +154,6 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.ssTableInd).byteSize() / ONE_LINE_SIZE; t++) { var inf = createRowInfo(info.ssTableInd, t); - if (inf.isDeletedData()) { - continue; - } - Entry row = getRow(inf); if (segComp.compare(row.key(), maxKey) < 0) { return inf; From 4c71c45c95f13cec8784393a536f3ee5ba58c110 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 01:25:18 +0300 Subject: [PATCH 31/57] was late, because one little test:( --- .../java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java | 2 -- .../java/ru/vk/itmo/test/kachmareugene/SSTablesController.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 79d9aec15..636c763dc 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -149,11 +149,9 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); - while (!mp.isEmpty() && !isBetween(minSStablesEntry.getKey())) { mp.remove(minSStablesEntry.getKey()); insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - minSStablesEntry = mp.firstEntry(); } return minSStablesEntry; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 7a11bbe8f..758636050 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -15,12 +15,12 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.SortedMap; import java.util.stream.Stream; -import java.util.Collections; import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; From f1e92a47b3e6e3dfbbb74b9442132c3a0a3cd8c8 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 01:58:31 +0300 Subject: [PATCH 32/57] refactored --- .../test/kachmareugene/SSTableIterator.java | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 636c763dc..0d2dbf782 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -97,49 +97,50 @@ private Entry moveAndGetOnce() { return ans; } - private void changeState() { - while (true) { + private boolean isChangedByNulls(Map.Entry minSStablesEntry) { + if (minSStablesEntry == null) { + keeper = moveAndGet(); + return true; + } + if (getHead() == null) { + updateMp(minSStablesEntry.getKey()); + keeper = controller.getRow(minSStablesEntry.getValue()); + if (keeper != null && keeper.value() == null) {return false;} + return true; + } + return false; + } - Map.Entry minSStablesEntry = getFirstMin(); + private boolean isChangedByStructures(Map.Entry minSStablesEntry) { + if (getHead() == null) {return false;} + int res = comp.compare(getHead().key(), minSStablesEntry.getKey()); - if (minSStablesEntry == null) { - keeper = moveAndGet(); - return; - } + if (res < 0) { + keeper = moveAndGet(); + return true; + } - var curHead = getHead(); + updateMp(minSStablesEntry.getKey()); + keeper = (res > 0) ? controller.getRow(minSStablesEntry.getValue()) : moveAndGetOnce(); - if (curHead == null) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - keeper = controller.getRow(minSStablesEntry.getValue()); - if (keeper != null && keeper.value() == null) { - continue; - } - return; - } + if (keeper.value() != null) { + return true; + } + return false; + } - int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); + private void changeState() { + while (true) { + Map.Entry minSStablesEntry = getFirstMin(); - if (res < 0) { - keeper = moveAndGet(); + if (isChangedByNulls(minSStablesEntry) || isChangedByStructures(minSStablesEntry)) { return; } - - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); - - if (res > 0) { - keeper = controller.getRow(minSStablesEntry.getValue()); - } else { - keeper = moveAndGetOnce(); - } - - if (keeper.value() != null) { - break; - } } } + private void updateMp(MemorySegment key) { + insertNew(controller.getNextInfo(mp.remove(key), to)); + } private boolean isBetween(MemorySegment who) { int res1 = comp.compare(from, who); From f7c97bbfb8f4c18b653ad03d948cdff02ca40e17 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 02:02:47 +0300 Subject: [PATCH 33/57] refactored --- .../itmo/test/kachmareugene/SSTableIterator.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 0d2dbf782..c104759bc 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -105,14 +105,18 @@ private boolean isChangedByNulls(Map.Entry minSSt if (getHead() == null) { updateMp(minSStablesEntry.getKey()); keeper = controller.getRow(minSStablesEntry.getValue()); - if (keeper != null && keeper.value() == null) {return false;} + if (keeper != null && keeper.value() == null) { + return false; + } return true; } return false; } private boolean isChangedByStructures(Map.Entry minSStablesEntry) { - if (getHead() == null) {return false;} + if (getHead() == null) { + return false; + } int res = comp.compare(getHead().key(), minSStablesEntry.getKey()); if (res < 0) { @@ -123,10 +127,7 @@ private boolean isChangedByStructures(Map.Entry m updateMp(minSStablesEntry.getKey()); keeper = (res > 0) ? controller.getRow(minSStablesEntry.getValue()) : moveAndGetOnce(); - if (keeper.value() != null) { - return true; - } - return false; + return keeper.value() != null; } private void changeState() { @@ -138,6 +139,7 @@ private void changeState() { } } } + private void updateMp(MemorySegment key) { insertNew(controller.getNextInfo(mp.remove(key), to)); } From b818d10d8e64997cd0d9d6138d86b6cbffcae042 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 02:06:41 +0300 Subject: [PATCH 34/57] refactored --- .../vk/itmo/test/kachmareugene/SSTableIterator.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index c104759bc..8c8c158e2 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -105,19 +105,17 @@ private boolean isChangedByNulls(Map.Entry minSSt if (getHead() == null) { updateMp(minSStablesEntry.getKey()); keeper = controller.getRow(minSStablesEntry.getValue()); - if (keeper != null && keeper.value() == null) { - return false; - } - return true; + return keeper != null && keeper.value() == null; } return false; } private boolean isChangedByStructures(Map.Entry minSStablesEntry) { - if (getHead() == null) { + var curHead = getHead(); + if (curHead == null) { return false; } - int res = comp.compare(getHead().key(), minSStablesEntry.getKey()); + int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); if (res < 0) { keeper = moveAndGet(); @@ -126,7 +124,6 @@ private boolean isChangedByStructures(Map.Entry m updateMp(minSStablesEntry.getKey()); keeper = (res > 0) ? controller.getRow(minSStablesEntry.getValue()) : moveAndGetOnce(); - return keeper.value() != null; } From da21ff1937a3b3b46682ad010c6827123e11026c Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Thu, 19 Oct 2023 02:14:27 +0300 Subject: [PATCH 35/57] refactored --- .../ru/vk/itmo/test/kachmareugene/SSTableIterator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 8c8c158e2..6d53ce9b3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -105,17 +105,17 @@ private boolean isChangedByNulls(Map.Entry minSSt if (getHead() == null) { updateMp(minSStablesEntry.getKey()); keeper = controller.getRow(minSStablesEntry.getValue()); - return keeper != null && keeper.value() == null; + return keeper == null || keeper.value() != null; } return false; } private boolean isChangedByStructures(Map.Entry minSStablesEntry) { - var curHead = getHead(); - if (curHead == null) { + var currHead = getHead(); + if (currHead == null) { return false; } - int res = comp.compare(curHead.key(), minSStablesEntry.getKey()); + int res = comp.compare(currHead.key(), minSStablesEntry.getKey()); if (res < 0) { keeper = moveAndGet(); From 04cf5653783e4793717d84b60ff0a7f94e5ebb9a Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 15:49:40 +0300 Subject: [PATCH 36/57] first commit for 4th --- .../itmo/test/kachmareugene/DaoFactory.java | 6 +- .../test/kachmareugene/DaoWithCompaction.java | 24 +++++ .../itmo/test/kachmareugene/InMemoryDao.java | 15 +++- .../test/kachmareugene/SSTableIterable.java | 28 ++++++ .../kachmareugene/SSTablesController.java | 90 ++++++++++++++++--- 5 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 74efad8e6..97962b00c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -9,7 +9,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@ru.vk.itmo.test.DaoFactory(stage = 3) +@ru.vk.itmo.test.DaoFactory(stage = 4) public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -34,11 +34,11 @@ public Entry fromBaseEntry(Entry baseEntry) { @Override public Dao> createDao() { - return new InMemoryDao(); + return new DaoWithCompaction(); } @Override public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config); + return new DaoWithCompaction(config); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java new file mode 100644 index 000000000..e1006defb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java @@ -0,0 +1,24 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Config; +import java.io.IOException; + +// TODO убрать поиск за лог +// TODO убрать дополнения в виде дат +// TODO считать отдельно длину файла и memtable - ошибка +public class DaoWithCompaction extends InMemoryDao { + public DaoWithCompaction() { + super(); + } + + public DaoWithCompaction(Config conf) { + super(conf); + } + + @Override + public void compact() throws IOException { + controller.dumpIterator(new SSTableIterable(getMemTable().values(), controller, null, null)); + closeMemTable(); + controller.deleteAllOldFiles(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index ab10a5a93..a0f11f803 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -17,7 +17,7 @@ public class InMemoryDao implements Dao> { private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); - private final SSTablesController controller; + protected final SSTablesController controller; public InMemoryDao() { this.controller = new SSTablesController(new MemSegComparatorNull()); @@ -66,7 +66,18 @@ public void upsert(Entry entry) { @Override public void close() throws IOException { - controller.dumpMemTableToSStable(mp); + try { + controller.dumpMemTableToSStable(mp); + } finally { + mp.clear(); + } + } + + protected void closeMemTable() { mp.clear(); } + protected SortedMap> getMemTable() { + return mp; + } + } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java new file mode 100644 index 000000000..ebfad19ab --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java @@ -0,0 +1,28 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collection; +import java.util.Iterator; +import java.util.SortedMap; + +public class SSTableIterable implements Iterable> { + private final Collection> memTable; + private final SSTablesController controller; + private final MemorySegment from; + private final MemorySegment to; + public SSTableIterable(Collection> it, SSTablesController controller, + MemorySegment from, MemorySegment to) { + memTable = it; + this.controller = controller; + + this.from = from; + this.to = to; + } + + @Override + public Iterator> iterator() { + return new SSTableIterator(memTable.iterator(), controller, from, to); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 758636050..66242d0e1 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -14,12 +14,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Set; -import java.util.SortedMap; +import java.util.*; import java.util.stream.Stream; import static java.nio.file.StandardOpenOption.CREATE; @@ -31,20 +26,23 @@ public class SSTablesController { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); private final List ssTables = new ArrayList<>(); private final List ssTablesIndexes = new ArrayList<>(); + private final List ssTablesPaths = new ArrayList<>(); + private final List ssTablesIndexesPaths = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; private static final String INDEX_COMMON_PREF = "index"; private final Arena arenaForReading = Arena.ofShared(); + private boolean isClosedArena = false; private final Comparator segComp; public SSTablesController(Path dir, Comparator com) { this.ssTablesDir = dir; this.segComp = com; - openFiles(dir, SS_TABLE_COMMON_PREF, ssTables); - openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes); + ssTablesPaths.addAll(openFiles(dir, SS_TABLE_COMMON_PREF, ssTables)); + ssTablesIndexesPaths.addAll(openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes)); } public SSTablesController(Comparator com) { @@ -52,7 +50,7 @@ public SSTablesController(Comparator com) { this.segComp = com; } - private void openFiles(Path dir, String fileNamePref, List storage) { + private List openFiles(Path dir, String fileNamePref, List storage) { try { Files.createDirectories(dir); } catch (IOException e) { @@ -69,6 +67,7 @@ private void openFiles(Path dir, String fileNamePref, List storag throw new UncheckedIOException(e); } }); + return list; } catch (IOException e) { throw new UncheckedIOException(e); } @@ -173,10 +172,56 @@ private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) return offset + data.byteSize(); } + public void dumpIterator(Iterable> iter) throws IOException { + Iterator> iter1 = iter.iterator(); + + if (ssTablesDir == null || !iter1.hasNext()) { + return; + } + LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); + Set options = Set.of(WRITE, READ, CREATE); + try (FileChannel ssTableChannel = + FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), options); + FileChannel indexChannel = + FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), options); + Arena saveArena = Arena.ofConfined()) { + + long ssTableLenght = 0L; + long indexLength = 0L; + int n = 0; + while (iter1.hasNext()) { + var seg = iter1.next(); + n++; + ssTableLenght += seg.key().byteSize() + seg.value().byteSize(); + indexLength += ONE_LINE_SIZE; + } + System.err.println(n); + long currOffsetSSTable = 0L; + long currOffsetIndex = 0L; + + MemorySegment mappedSSTable = ssTableChannel.map( + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); + + MemorySegment mappedIndex = indexChannel.map( + FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); + + + for (Entry kv : iter) { + currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); + currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + + currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); + currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); + } + } + } + public void dumpMemTableToSStable(SortedMap> mp) throws IOException { if (ssTablesDir == null || mp.isEmpty()) { - arenaForReading.close(); + closeArena(); return; } LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); @@ -214,7 +259,7 @@ public void dumpMemTableToSStable(SortedMap> currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); } } finally { - arenaForReading.close(); + closeArena(); } } @@ -232,4 +277,27 @@ private MemorySegment getValueOrNull(Entry kv) { } return value; } + + private void closeArena() { + if (!isClosedArena) { + arenaForReading.close(); + } + isClosedArena = true; + } + + public void deleteAllOldFiles() throws IOException { + closeArena(); + deleteFiles(ssTablesPaths); + if (!ssTables.isEmpty()) { + System.out.println(ssTables.get(0).byteSize()); + System.out.println(ssTablesIndexes.get(0).byteSize()); + } + deleteFiles(ssTablesIndexesPaths); + } + private void deleteFiles(List files) throws IOException { + for (Path file : files) { + Files.deleteIfExists(file); + } + files.clear(); + } } From 6707aada9e22f2a84acf1cf97206e7448e6453cd Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 16:24:36 +0300 Subject: [PATCH 37/57] mmm, codeclimate --- .../vk/itmo/test/kachmareugene/InMemoryDao.java | 1 + .../itmo/test/kachmareugene/SSTableIterable.java | 1 - .../test/kachmareugene/SSTablesController.java | 15 +++++++-------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index a0f11f803..049b602bf 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -76,6 +76,7 @@ public void close() throws IOException { protected void closeMemTable() { mp.clear(); } + protected SortedMap> getMemTable() { return mp; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java index ebfad19ab..fb8103050 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java @@ -5,7 +5,6 @@ import java.lang.foreign.MemorySegment; import java.util.Collection; import java.util.Iterator; -import java.util.SortedMap; public class SSTableIterable implements Iterable> { private final Collection> memTable; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 66242d0e1..6724fae7e 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -14,7 +14,13 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.Iterator; +import java.util.SortedMap; import java.util.stream.Stream; import static java.nio.file.StandardOpenOption.CREATE; @@ -29,7 +35,6 @@ public class SSTablesController { private final List ssTablesPaths = new ArrayList<>(); private final List ssTablesIndexesPaths = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; - // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; private static final String INDEX_COMMON_PREF = "index"; @@ -158,7 +163,6 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return inf; } } - return null; } @@ -195,7 +199,6 @@ public void dumpIterator(Iterable> iter) throws IOException ssTableLenght += seg.key().byteSize() + seg.value().byteSize(); indexLength += ONE_LINE_SIZE; } - System.err.println(n); long currOffsetSSTable = 0L; long currOffsetIndex = 0L; @@ -288,10 +291,6 @@ private void closeArena() { public void deleteAllOldFiles() throws IOException { closeArena(); deleteFiles(ssTablesPaths); - if (!ssTables.isEmpty()) { - System.out.println(ssTables.get(0).byteSize()); - System.out.println(ssTablesIndexes.get(0).byteSize()); - } deleteFiles(ssTablesIndexesPaths); } private void deleteFiles(List files) throws IOException { From f499825b8923013f3273e66aea8b3def7687e832 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 16:37:09 +0300 Subject: [PATCH 38/57] mmm, codeclimate --- .../java/ru/vk/itmo/test/kachmareugene/SSTablesController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 6724fae7e..ba0cef86a 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -192,10 +192,8 @@ public void dumpIterator(Iterable> iter) throws IOException long ssTableLenght = 0L; long indexLength = 0L; - int n = 0; while (iter1.hasNext()) { var seg = iter1.next(); - n++; ssTableLenght += seg.key().byteSize() + seg.value().byteSize(); indexLength += ONE_LINE_SIZE; } From f48b5f2c7c3f965e6d55b30c31987379c7aa50a4 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 16:54:45 +0300 Subject: [PATCH 39/57] mmm, codeclimate --- src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java | 1 + .../java/ru/vk/itmo/test/kachmareugene/SSTablesController.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java index fb8103050..8d315576b 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java @@ -11,6 +11,7 @@ public class SSTableIterable implements Iterable> { private final SSTablesController controller; private final MemorySegment from; private final MemorySegment to; + public SSTableIterable(Collection> it, SSTablesController controller, MemorySegment from, MemorySegment to) { memTable = it; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index ba0cef86a..b696e6fbd 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -291,6 +291,7 @@ public void deleteAllOldFiles() throws IOException { deleteFiles(ssTablesPaths); deleteFiles(ssTablesIndexesPaths); } + private void deleteFiles(List files) throws IOException { for (Path file : files) { Files.deleteIfExists(file); From cfd528b8de0f7b87965af2770b8b70d89c44c7d9 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:01:37 +0300 Subject: [PATCH 40/57] mmm, codeclimate --- .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index b696e6fbd..8ded6e28c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -17,9 +17,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.Iterator; import java.util.SortedMap; import java.util.stream.Stream; @@ -39,7 +39,7 @@ public class SSTablesController { private static final long ONE_LINE_SIZE = 4 * Long.BYTES; private static final String INDEX_COMMON_PREF = "index"; private final Arena arenaForReading = Arena.ofShared(); - private boolean isClosedArena = false; + private boolean isClosedArena; private final Comparator segComp; public SSTablesController(Path dir, Comparator com) { @@ -206,7 +206,6 @@ public void dumpIterator(Iterable> iter) throws IOException MemorySegment mappedIndex = indexChannel.map( FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); - for (Entry kv : iter) { currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); From 510d890f89bdb2c75091b9c2edc47aa6168880c1 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:17:57 +0300 Subject: [PATCH 41/57] mmm, codeclimate --- .../java/ru/vk/itmo/test/kachmareugene/SSTablesController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 8ded6e28c..0c51276fd 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -35,7 +35,7 @@ public class SSTablesController { private final List ssTablesPaths = new ArrayList<>(); private final List ssTablesIndexesPaths = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; - // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen + // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; private static final String INDEX_COMMON_PREF = "index"; private final Arena arenaForReading = Arena.ofShared(); From e643993dd72f6473b16be8cf741f7ca584bb6981 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:27:16 +0300 Subject: [PATCH 42/57] fixed --- .../ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java | 3 --- .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java index e1006defb..b18717438 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java @@ -3,9 +3,6 @@ import ru.vk.itmo.Config; import java.io.IOException; -// TODO убрать поиск за лог -// TODO убрать дополнения в виде дат -// TODO считать отдельно длину файла и memtable - ошибка public class DaoWithCompaction extends InMemoryDao { public DaoWithCompaction() { super(); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 0c51276fd..475fe8020 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -108,7 +108,11 @@ public List firstGreaterKeys(MemorySegment key) { List ans = new ArrayList<>(); for (int i = ssTables.size() - 1; i >= 0; i--) { - long entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + long entryIndexesLine = 0; + if (key != null) { + entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + } + if (entryIndexesLine < 0) { continue; } From 0bc60e274dd6aa5e8f3568d087038677bf399bd0 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:29:01 +0300 Subject: [PATCH 43/57] mmm, codeclimate --- .../java/ru/vk/itmo/test/kachmareugene/SSTablesController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 475fe8020..1699f19d1 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -93,7 +93,6 @@ private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData while (r - l > 1) { long mid = (l + r) / 2; - if (greaterThen(mappedIndex, mid * ONE_LINE_SIZE, mappedData, key)) { l = mid; } else { @@ -112,7 +111,6 @@ public List firstGreaterKeys(MemorySegment key) { if (key != null) { entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); } - if (entryIndexesLine < 0) { continue; } From 52cc05776025610f4bfaf3831c5d6423f0127c61 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:49:49 +0300 Subject: [PATCH 44/57] mmm, codeclimate --- .../itmo/test/kachmareugene/InMemoryDao.java | 2 +- .../kachmareugene/SSTablesController.java | 70 +++---------------- .../ru/vk/itmo/test/kachmareugene/Utils.java | 16 +++++ 3 files changed, 25 insertions(+), 63 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 049b602bf..b637378b7 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -67,7 +67,7 @@ public void upsert(Entry entry) { @Override public void close() throws IOException { try { - controller.dumpMemTableToSStable(mp); + controller.dumpIterator(mp.values()); } finally { mp.clear(); } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 1699f19d1..923cbf059 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -20,7 +20,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.SortedMap; import java.util.stream.Stream; import static java.nio.file.StandardOpenOption.CREATE; @@ -168,20 +167,11 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return null; } - private long dumpLong(MemorySegment mapped, long value, long offset) { - mapped.set(ValueLayout.JAVA_LONG, offset, value); - return offset + Long.BYTES; - } - - private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { - MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); - return offset + data.byteSize(); - } - public void dumpIterator(Iterable> iter) throws IOException { Iterator> iter1 = iter.iterator(); if (ssTablesDir == null || !iter1.hasNext()) { + closeArena(); return; } LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); @@ -196,7 +186,7 @@ public void dumpIterator(Iterable> iter) throws IOException long indexLength = 0L; while (iter1.hasNext()) { var seg = iter1.next(); - ssTableLenght += seg.key().byteSize() + seg.value().byteSize(); + ssTableLenght += seg.key().byteSize() + getValueOrNull(seg).byteSize(); indexLength += ONE_LINE_SIZE; } long currOffsetSSTable = 0L; @@ -204,61 +194,17 @@ public void dumpIterator(Iterable> iter) throws IOException MemorySegment mappedSSTable = ssTableChannel.map( FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); - MemorySegment mappedIndex = indexChannel.map( FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); for (Entry kv : iter) { - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); - - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); - } - } - } - - public void dumpMemTableToSStable(SortedMap> mp) throws IOException { - - if (ssTablesDir == null || mp.isEmpty()) { - closeArena(); - return; - } - LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); - Set options = Set.of(WRITE, READ, CREATE); - try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), options); - FileChannel indexChannel = - FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), options); - Arena saveArena = Arena.ofConfined()) { - - long ssTableLenght = 0L; - long indexLength = mp.size() * ONE_LINE_SIZE; - - for (var kv : mp.values()) { - ssTableLenght += - kv.key().byteSize() + getValueOrNull(kv).byteSize(); - } - - long currOffsetSSTable = 0L; - long currOffsetIndex = 0L; - - MemorySegment mappedSSTable = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); - - MemorySegment mappedIndex = indexChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); - - for (var kv : mp.values()) { - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + currOffsetIndex = Utils.dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); + currOffsetIndex = Utils.dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); + currOffsetIndex = Utils.dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); + currOffsetIndex = Utils.dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); } } finally { closeArena(); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java new file mode 100644 index 000000000..35840ebee --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -0,0 +1,16 @@ +package ru.vk.itmo.test.kachmareugene; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public class Utils { + public static long dumpLong(MemorySegment mapped, long value, long offset) { + mapped.set(ValueLayout.JAVA_LONG, offset, value); + return offset + Long.BYTES; + } + public static long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { + MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); + return offset + data.byteSize(); + } + +} From 95c04cefd0c543eacfb2bac46cd1268892bead0b Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:56:24 +0300 Subject: [PATCH 45/57] mmm, codeclimate --- .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 2 +- src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 923cbf059..5e3b2059f 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -101,7 +101,7 @@ private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData return r == (mappedIndex.byteSize() / ONE_LINE_SIZE) ? -1 : r; } - //return - List ordered form the latest created sstable to the first. + //return - List ordered form the latest created sstable to the first. public List firstGreaterKeys(MemorySegment key) { List ans = new ArrayList<>(); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 35840ebee..b89b3aeef 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -4,10 +4,14 @@ import java.lang.foreign.ValueLayout; public class Utils { + private Utils() { + + } public static long dumpLong(MemorySegment mapped, long value, long offset) { mapped.set(ValueLayout.JAVA_LONG, offset, value); return offset + Long.BYTES; } + public static long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); return offset + data.byteSize(); From dec80cb6caa1f488b6107f429f14f0b8ec464a6a Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 17:58:29 +0300 Subject: [PATCH 46/57] mmm, codeclimate --- src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index b89b3aeef..3d1ae8977 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -3,10 +3,11 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -public class Utils { +public final class Utils { private Utils() { } + public static long dumpLong(MemorySegment mapped, long value, long offset) { mapped.set(ValueLayout.JAVA_LONG, offset, value); return offset + Long.BYTES; From b7cf38e2b1d10ce92607c0e67952e3d423787f78 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 18:42:36 +0300 Subject: [PATCH 47/57] fixed timestamps --- .../kachmareugene/SSTablesController.java | 20 ++++++------ .../ru/vk/itmo/test/kachmareugene/Utils.java | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 5e3b2059f..3780a00bc 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -11,11 +11,7 @@ import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -28,7 +24,6 @@ public class SSTablesController { private final Path ssTablesDir; - private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); private final List ssTables = new ArrayList<>(); private final List ssTablesIndexes = new ArrayList<>(); private final List ssTablesPaths = new ArrayList<>(); @@ -63,7 +58,7 @@ private List openFiles(Path dir, String fileNamePref, List try (Stream tabels = Files.find(dir, 1, (path, ignore) -> path.getFileName().toString().startsWith(fileNamePref))) { final List list = new ArrayList<>(tabels.toList()); - Collections.sort(list); + Utils.sortByNames(list, fileNamePref); list.forEach(t -> { try (FileChannel channel = FileChannel.open(t, READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); @@ -174,12 +169,19 @@ public void dumpIterator(Iterable> iter) throws IOException closeArena(); return; } - LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); Set options = Set.of(WRITE, READ, CREATE); try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), options); + FileChannel.open(ssTablesDir + .resolve(SS_TABLE_COMMON_PREF + + String.valueOf( + Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1)), + options); FileChannel indexChannel = - FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), options); + FileChannel.open(ssTablesDir + .resolve(INDEX_COMMON_PREF + + String.valueOf( + Utils.getMaxNumberOfFile(ssTablesDir, INDEX_COMMON_PREF) + 1)), + options); Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 3d1ae8977..804ec39df 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -1,7 +1,15 @@ package ru.vk.itmo.test.kachmareugene; +import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + public final class Utils { private Utils() { @@ -18,4 +26,27 @@ public static long dumpSegment(MemorySegment mapped, MemorySegment data, long of return offset + data.byteSize(); } + public static long getNumberFromFileName(Path pathToFile, String prefix) { + return Long.parseLong(pathToFile.getFileName().toString().substring(prefix.length())); + } + + public static void sortByNames(List l, String prefix) { + l.sort(Comparator.comparingLong(s -> getNumberFromFileName(s, prefix))); + } + + public static long getMaxNumberOfFile(Path dir, String prefix) { + try (Stream tabels = Files.find(dir, 1, + (path, ignore) -> path.getFileName().toString().startsWith(prefix))) { + final List list = tabels.toList(); + long maxi = 0; + for (Path p : list) { + if (getNumberFromFileName(p, prefix) > maxi) { + maxi = getNumberFromFileName(p, prefix); + } + } + return maxi; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } From 69287ff0e2382a2511b304c9432b00019ad3e5a1 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 1 Nov 2023 18:53:33 +0300 Subject: [PATCH 48/57] fixed timestamps --- .../itmo/test/kachmareugene/SSTablesController.java | 11 +++-------- .../java/ru/vk/itmo/test/kachmareugene/Utils.java | 1 - 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 3780a00bc..dd75e7374 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -169,19 +169,14 @@ public void dumpIterator(Iterable> iter) throws IOException closeArena(); return; } + String suff = String.valueOf(Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1); Set options = Set.of(WRITE, READ, CREATE); try (FileChannel ssTableChannel = FileChannel.open(ssTablesDir - .resolve(SS_TABLE_COMMON_PREF + - String.valueOf( - Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1)), - options); + .resolve(SS_TABLE_COMMON_PREF + suff), options); FileChannel indexChannel = FileChannel.open(ssTablesDir - .resolve(INDEX_COMMON_PREF + - String.valueOf( - Utils.getMaxNumberOfFile(ssTablesDir, INDEX_COMMON_PREF) + 1)), - options); + .resolve(INDEX_COMMON_PREF + suff), options); Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 804ec39df..20df7ef73 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -10,7 +10,6 @@ import java.util.List; import java.util.stream.Stream; - public final class Utils { private Utils() { From 64923591259386f41c537ab59c1429eda44fd7a2 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Fri, 17 Nov 2023 02:08:28 +0300 Subject: [PATCH 49/57] upload changes --- .../itmo/test/kachmareugene/DaoFactory.java | 5 - .../itmo/test/kachmareugene/InMemoryDao.java | 11 +- .../kachmareugene/MemSegComparatorNull.java | 3 - .../test/kachmareugene/SSTableIterator.java | 32 ++--- .../test/kachmareugene/SSTableRowInfo.java | 15 ++- .../kachmareugene/SSTablesController.java | 127 +++++++++--------- .../ru/vk/itmo/test/kachmareugene/Utils.java | 68 ++++++++++ 7 files changed, 163 insertions(+), 98 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 057df585b..97962b00c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -41,9 +41,4 @@ public Dao> createDao() { public Dao> createDao(Config config) throws IOException { return new DaoWithCompaction(config); } - - @Override - public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config); - } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index d2f4ccfca..b637378b7 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -46,12 +46,11 @@ public Iterator> get(MemorySegment from, MemorySegment to) @Override public Entry get(MemorySegment key) { - if (key == null) { - return null; - } - Entry value = mp.get(key); - if (value != null) { - return value.value() == null ? null : value; + if (mp.containsKey(key)) { + if (mp.get(key).value() == null) { + return null; + } + return mp.get(key); } var res = controller.getRow(controller.searchInSStables(key)); if (res == null) { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java index a7e1e6b6f..eaff57f3c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java @@ -5,9 +5,6 @@ public class MemSegComparatorNull extends MemorySegmentComparator { @Override public int compare(MemorySegment segment1, MemorySegment segment2) { - if (segment1 == null && segment2 == null) { - throw new IllegalArgumentException("Incomparable null and null"); - } if (segment1 == null) { return -1; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 6e096bc94..6d53ce9b3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -23,7 +23,7 @@ public class SSTableIterator implements Iterator> { public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { - this.memTableIterator = it; + memTableIterator = it; this.controller = controller; this.from = from; @@ -32,24 +32,26 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } - private void insertNew(final SSTableRowInfo info) { - var curInfo = info; - Entry kv = controller.getRow(curInfo); + private void insertNew(SSTableRowInfo info) { + Entry kv = controller.getRow(info); - while (kv != null) { - SSTableRowInfo old = mp.putIfAbsent(kv.key(), curInfo); - if (old == null) { - return; - } + if (kv == null) { + return; + } - SSTableRowInfo oldInfo = old.ssTableInd > curInfo.ssTableInd ? curInfo : old; - SSTableRowInfo newInfo = old.ssTableInd < curInfo.ssTableInd ? curInfo : old; + if (!mp.containsKey(kv.key())) { + mp.put(kv.key(), info); + return; + } + SSTableRowInfo old = mp.get(kv.key()); - mp.put(controller.getRow(newInfo).key(), newInfo); + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; + SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; - curInfo = controller.getNextInfo(oldInfo, to); - kv = controller.getRow(curInfo); - } + mp.put(controller.getRow(newInfo).key(), newInfo); + + // tail recursion + insertNew(controller.getNextInfo(oldInfo, to)); } private void positioningIterator() { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 697d9affb..309a26400 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -1,13 +1,12 @@ package ru.vk.itmo.test.kachmareugene; public class SSTableRowInfo { - - final long keyOffset; - final long valueOffset; - final long keySize; - final long rowShift; + long keyOffset; + long valueOffset; + long keySize; + long rowShift; private final long valueSize; - final int ssTableInd; + int ssTableInd; public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift) { @@ -26,4 +25,8 @@ public boolean isDeletedData() { public long getValueSize() { return valueSize; } + + public long totalShift() { + return keyOffset + keySize + valueSize; + } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 45cc7b82b..cf907d2e1 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -8,9 +8,7 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.OpenOption; -import java.nio.file.Path; +import java.nio.file.*; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; @@ -21,17 +19,15 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; import static java.nio.file.StandardOpenOption.WRITE; +import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; public class SSTablesController { private final Path ssTablesDir; private final List ssTables = new ArrayList<>(); - private final List ssTablesIndexes = new ArrayList<>(); private final List ssTablesPaths = new ArrayList<>(); - private final List ssTablesIndexesPaths = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; - private static final String INDEX_COMMON_PREF = "index"; private final Arena arenaForReading = Arena.ofShared(); private boolean isClosedArena; private final Comparator segComp; @@ -41,7 +37,6 @@ public SSTablesController(Path dir, Comparator com) { this.segComp = com; ssTablesPaths.addAll(openFiles(dir, SS_TABLE_COMMON_PREF, ssTables)); - ssTablesIndexesPaths.addAll(openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes)); } public SSTablesController(Comparator com) { @@ -72,27 +67,27 @@ private List openFiles(Path dir, String fileNamePref, List } } - private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, - MemorySegment mappedData, MemorySegment key) { - long offset = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset); - long size = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset + Long.BYTES); - return segComp.compare(key, mappedData.asSlice(offset, size)) > 0; + private boolean greaterThen(long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; } //Gives offset for line in index file - private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { + private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { long l = -1; - long r = mappedIndex.byteSize() / ONE_LINE_SIZE; + long r = getNumberOfEntries(mapped); while (r - l > 1) { long mid = (l + r) / 2; - if (greaterThen(mappedIndex, mid * ONE_LINE_SIZE, mappedData, key)) { + SSTableRowInfo info = createRowInfo(ind, mid); + if (greaterThen(info.keyOffset, info.keySize, mapped, key)) { l = mid; } else { r = mid; } } - return r == (mappedIndex.byteSize() / ONE_LINE_SIZE) ? -1 : r; + return r == getNumberOfEntries(mapped) ? -1 : r; } //return - List ordered form the latest created sstable to the first. @@ -102,7 +97,7 @@ public List firstGreaterKeys(MemorySegment key) { for (int i = ssTables.size() - 1; i >= 0; i--) { long entryIndexesLine = 0; if (key != null) { - entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); } if (entryIndexesLine < 0) { continue; @@ -112,18 +107,18 @@ public List firstGreaterKeys(MemorySegment key) { return ans; } - private SSTableRowInfo createRowInfo(int ind, long rowShift) { - long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE); - long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE + Long.BYTES); + private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { + long start = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES); + long size = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES * 2); - long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 2); - long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 3); - return new SSTableRowInfo(start, size, start1, size1, ind, rowShift); + long start1 = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED,rowIndex * ONE_LINE_SIZE + Long.BYTES * 3); + long size1 = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED,rowIndex * ONE_LINE_SIZE + Long.BYTES * 4); + return new SSTableRowInfo(start, size, start1, size1, ind, rowIndex); } public SSTableRowInfo searchInSStables(MemorySegment key) { - for (int i = ssTablesIndexes.size() - 1; i >= 0; i--) { - long ind = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + for (int i = ssTables.size() - 1; i >= 0; i--) { + long ind = searchKeyInFile(i, ssTables.get(i), key); if (ind >= 0) { return createRowInfo(i, ind); } @@ -150,7 +145,7 @@ public Entry getRow(SSTableRowInfo info) { * Ignores deleted values. */ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { - for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.ssTableInd).byteSize() / ONE_LINE_SIZE; t++) { + for (long t = info.rowShift + 1; t < getNumberOfEntries(ssTables.get(info.ssTableInd)); t++) { var inf = createRowInfo(info.ssTableInd, t); Entry row = getRow(inf); @@ -161,6 +156,10 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return null; } + private long getNumberOfEntries(MemorySegment memSeg) { + return memSeg.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + public void dumpIterator(Iterable> iter) throws IOException { Iterator> iter1 = iter.iterator(); @@ -170,57 +169,67 @@ public void dumpIterator(Iterable> iter) throws IOException } String suff = String.valueOf(Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1); Set options = Set.of(WRITE, READ, CREATE); + + final Path tmpFile = ssTablesDir.resolve("data.tmp"); + final Path targetFile = ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff); + + try { + Files.createFile(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff)); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + + try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir - .resolve(SS_TABLE_COMMON_PREF + suff), options); - FileChannel indexChannel = - FileChannel.open(ssTablesDir - .resolve(INDEX_COMMON_PREF + suff), options); + FileChannel.open(tmpFile, options); Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; long indexLength = 0L; + while (iter1.hasNext()) { var seg = iter1.next(); ssTableLenght += seg.key().byteSize() + getValueOrNull(seg).byteSize(); indexLength += ONE_LINE_SIZE; } long currOffsetSSTable = 0L; - long currOffsetIndex = 0L; MemorySegment mappedSSTable = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); - MemorySegment mappedIndex = indexChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght + indexLength + Long.BYTES, + saveArena); + + currOffsetSSTable = Utils.dumpLong(mappedSSTable, indexLength / ONE_LINE_SIZE, currOffsetSSTable); + + long shiftForData = indexLength + Long.BYTES; for (Entry kv : iter) { - currOffsetIndex = Utils.dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = Utils.dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); - currOffsetIndex = Utils.dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + // key offset + currOffsetSSTable = Utils.dumpLong(mappedSSTable, shiftForData, currOffsetSSTable); + // key length + currOffsetSSTable = Utils.dumpLong(mappedSSTable, kv.key().byteSize(), currOffsetSSTable); + shiftForData += kv.key().byteSize(); + + // value offset + currOffsetSSTable = Utils.dumpLong(mappedSSTable, shiftForData, currOffsetSSTable); + // value length + currOffsetSSTable = Utils.dumpLong(mappedSSTable, Utils.rightByteSize(kv), currOffsetSSTable); + shiftForData += getValueOrNull(kv).byteSize(); + } - currOffsetIndex = Utils.dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); + for (Entry kv : iter) { + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); currOffsetSSTable = Utils.dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); - currOffsetIndex = Utils.dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); } + + Files.deleteIfExists(targetFile); + + Files.move(tmpFile, targetFile, StandardCopyOption.ATOMIC_MOVE); + } finally { closeArena(); } } - private long rightByteSize(Entry memSeg) { - if (memSeg.value() == null) { - return -1; - } - return memSeg.value().byteSize(); - } - - private MemorySegment getValueOrNull(Entry kv) { - MemorySegment value = kv.value(); - if (kv.value() == null) { - value = MemorySegment.NULL; - } - return value; - } private void closeArena() { if (!isClosedArena) { arenaForReading.close(); @@ -230,14 +239,6 @@ private void closeArena() { public void deleteAllOldFiles() throws IOException { closeArena(); - deleteFiles(ssTablesPaths); - deleteFiles(ssTablesIndexesPaths); - } - - private void deleteFiles(List files) throws IOException { - for (Path file : files) { - Files.deleteIfExists(file); - } - files.clear(); + Utils.deleteFiles(ssTablesPaths); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 20df7ef73..4797897f6 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -1,11 +1,16 @@ package ru.vk.itmo.test.kachmareugene; +import ru.vk.itmo.Entry; + import java.io.IOException; import java.io.UncheckedIOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.stream.Stream; @@ -48,4 +53,67 @@ public static long getMaxNumberOfFile(Path dir, String prefix) { throw new UncheckedIOException(e); } } + + public static void deleteFiles(List files) throws IOException { + for (Path file : files) { + Files.deleteIfExists(file); + } + files.clear(); + } + + public static long rightByteSize(Entry memSeg) { + if (memSeg.value() == null) { + return -1; + } + return memSeg.value().byteSize(); + } + + public static MemorySegment getValueOrNull(Entry kv) { + MemorySegment value = kv.value(); + if (kv.value() == null) { + value = MemorySegment.NULL; + } + return value; + } + + public static void finalizeCompaction(Path storagePath, String SSTablePrefix) throws IOException { + Path compactionFile = compactionFile(storagePath); + try (Stream stream = Files.find(storagePath, 1, + (path, attrs) -> path.getFileName().toString().startsWith(SSTablePrefix))) { + stream.forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + Path indexTmp = storagePath.resolve("index.tmp"); + Path indexFile = storagePath.resolve("index.idx"); + + Files.deleteIfExists(indexFile); + Files.deleteIfExists(indexTmp); + + boolean noData = Files.size(compactionFile) == 0; + + Files.write( + indexTmp, + noData ? Collections.emptyList() : Collections.singleton(SSTablePrefix + "0"), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); + if (noData) { + Files.delete(compactionFile); + } else { + Files.move(compactionFile, storagePath.resolve(SSTablePrefix + "0"), StandardCopyOption.ATOMIC_MOVE); + } + } + + private static Path compactionFile(Path storagePath) { + return storagePath.resolve("compaction"); + } } From 07da6007e54204b5cf4e88cf9939e6b68ff59af5 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 29 Nov 2023 14:28:15 +0300 Subject: [PATCH 50/57] fixed --- .../itmo/test/kachmareugene/DaoFactory.java | 2 +- .../itmo/test/kachmareugene/InMemoryDao.java | 12 +++--- .../kachmareugene/SSTablesController.java | 10 ++--- .../ru/vk/itmo/test/kachmareugene/Utils.java | 37 ------------------- 4 files changed, 11 insertions(+), 50 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 97962b00c..bba99fa88 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -9,7 +9,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@ru.vk.itmo.test.DaoFactory(stage = 4) +@ru.vk.itmo.test.DaoFactory(stage = 4, week = 1111) public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index b637378b7..516bb75d5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -46,12 +46,14 @@ public Iterator> get(MemorySegment from, MemorySegment to) @Override public Entry get(MemorySegment key) { - if (mp.containsKey(key)) { - if (mp.get(key).value() == null) { - return null; - } - return mp.get(key); + if (key == null) { + return null; + } + Entry value = mp.get(key); + if (value != null) { + return value.value() == null ? null : value; } + var res = controller.getRow(controller.searchInSStables(key)); if (res == null) { return null; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index cf907d2e1..c48438bd9 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -16,9 +16,7 @@ import java.util.Set; import java.util.stream.Stream; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.WRITE; +import static java.nio.file.StandardOpenOption.*; import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; public class SSTablesController { @@ -28,6 +26,7 @@ public class SSTablesController { private static final String SS_TABLE_COMMON_PREF = "ssTable"; // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; + private static final Set options = Set.of(WRITE, READ, CREATE, TRUNCATE_EXISTING); private final Arena arenaForReading = Arena.ofShared(); private boolean isClosedArena; private final Comparator segComp; @@ -168,7 +167,6 @@ public void dumpIterator(Iterable> iter) throws IOException return; } String suff = String.valueOf(Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1); - Set options = Set.of(WRITE, READ, CREATE); final Path tmpFile = ssTablesDir.resolve("data.tmp"); final Path targetFile = ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff); @@ -221,9 +219,7 @@ public void dumpIterator(Iterable> iter) throws IOException currOffsetSSTable = Utils.dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); } - Files.deleteIfExists(targetFile); - - Files.move(tmpFile, targetFile, StandardCopyOption.ATOMIC_MOVE); + Files.move(tmpFile, targetFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } finally { closeArena(); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 4797897f6..182cc89a7 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -76,43 +76,6 @@ public static MemorySegment getValueOrNull(Entry kv) { return value; } - public static void finalizeCompaction(Path storagePath, String SSTablePrefix) throws IOException { - Path compactionFile = compactionFile(storagePath); - try (Stream stream = Files.find(storagePath, 1, - (path, attrs) -> path.getFileName().toString().startsWith(SSTablePrefix))) { - stream.forEach(p -> { - try { - Files.delete(p); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } - - Path indexTmp = storagePath.resolve("index.tmp"); - Path indexFile = storagePath.resolve("index.idx"); - - Files.deleteIfExists(indexFile); - Files.deleteIfExists(indexTmp); - - boolean noData = Files.size(compactionFile) == 0; - - Files.write( - indexTmp, - noData ? Collections.emptyList() : Collections.singleton(SSTablePrefix + "0"), - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ); - - Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); - if (noData) { - Files.delete(compactionFile); - } else { - Files.move(compactionFile, storagePath.resolve(SSTablePrefix + "0"), StandardCopyOption.ATOMIC_MOVE); - } - } - private static Path compactionFile(Path storagePath) { return storagePath.resolve("compaction"); } From cbbc475815e44497112340d6e027fcbedd00ea63 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 29 Nov 2023 14:31:44 +0300 Subject: [PATCH 51/57] minor --- src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 182cc89a7..7b52ee192 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -75,8 +75,4 @@ public static MemorySegment getValueOrNull(Entry kv) { } return value; } - - private static Path compactionFile(Path storagePath) { - return storagePath.resolve("compaction"); - } } From 115ba3c99a16d470356b9cac6507ea5687fe00f9 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 29 Nov 2023 14:36:04 +0300 Subject: [PATCH 52/57] code cimate --- .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 7 +++++-- src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index c48438bd9..348ce9f4d 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -8,7 +8,11 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; -import java.nio.file.*; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; @@ -177,7 +181,6 @@ public void dumpIterator(Iterable> iter) throws IOException // it is ok, actually it is normal state } - try (FileChannel ssTableChannel = FileChannel.open(tmpFile, options); Arena saveArena = Arena.ofConfined()) { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 7b52ee192..77886d9d3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -8,9 +8,6 @@ import java.lang.foreign.ValueLayout; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.stream.Stream; From 89cb2dc3676876659da00959510fe0152ea24428 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 29 Nov 2023 14:38:18 +0300 Subject: [PATCH 53/57] code cimate --- .../ru/vk/itmo/test/kachmareugene/SSTablesController.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 348ce9f4d..e8c46f7e4 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -20,7 +20,10 @@ import java.util.Set; import java.util.stream.Stream; -import static java.nio.file.StandardOpenOption.*; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; public class SSTablesController { From c554338355eeaf15219dd25219551611dccc89b4 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 29 Nov 2023 14:44:09 +0300 Subject: [PATCH 54/57] code cimate --- .../vk/itmo/test/kachmareugene/SSTablesController.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index e8c46f7e4..7341db9e3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -13,6 +13,7 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; @@ -20,10 +21,6 @@ import java.util.Set; import java.util.stream.Stream; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.READ; -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; -import static java.nio.file.StandardOpenOption.WRITE; import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; public class SSTablesController { @@ -33,7 +30,8 @@ public class SSTablesController { private static final String SS_TABLE_COMMON_PREF = "ssTable"; // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; - private static final Set options = Set.of(WRITE, READ, CREATE, TRUNCATE_EXISTING); + private static final Set options = Set.of(StandardOpenOption.WRITE, StandardOpenOption.READ, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); private final Arena arenaForReading = Arena.ofShared(); private boolean isClosedArena; private final Comparator segComp; @@ -61,7 +59,7 @@ private List openFiles(Path dir, String fileNamePref, List final List list = new ArrayList<>(tabels.toList()); Utils.sortByNames(list, fileNamePref); list.forEach(t -> { - try (FileChannel channel = FileChannel.open(t, READ)) { + try (FileChannel channel = FileChannel.open(t, StandardOpenOption.READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); } catch (IOException e) { throw new UncheckedIOException(e); From a5eb57002a1a8fd367211b567d89a3502d725a31 Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Wed, 29 Nov 2023 14:47:13 +0300 Subject: [PATCH 55/57] code cimate --- .../java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java index 8d315576b..9c875c2c2 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java @@ -14,7 +14,7 @@ public class SSTableIterable implements Iterable> { public SSTableIterable(Collection> it, SSTablesController controller, MemorySegment from, MemorySegment to) { - memTable = it; + this.memTable = it; this.controller = controller; this.from = from; From 31c85090f31cc4a6dbf1eb169289a3afd92771ca Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 4 Dec 2023 19:16:38 +0300 Subject: [PATCH 56/57] add aditional task --- .../test/kachmareugene/DaoWithCompaction.java | 4 + .../itmo/test/kachmareugene/InMemoryDao.java | 9 + .../test/kachmareugene/IteratorUtils.java | 35 ++++ .../test/kachmareugene/SSTableIterator.java | 55 +++--- .../test/kachmareugene/SSTableRowInfo.java | 13 +- .../kachmareugene/SSTablesController.java | 49 +++++- .../java/ru/vk/itmo/ReverseIteratorTest.java | 158 ++++++++++++++++++ 7 files changed, 293 insertions(+), 30 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java create mode 100644 src/test/java/ru/vk/itmo/ReverseIteratorTest.java diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java index b18717438..e0750d132 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java @@ -1,7 +1,11 @@ package ru.vk.itmo.test.kachmareugene; import ru.vk.itmo.Config; +import ru.vk.itmo.Entry; + import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.util.Iterator; public class DaoWithCompaction extends InMemoryDao { public DaoWithCompaction() { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 516bb75d5..1237567b9 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -29,6 +29,10 @@ public InMemoryDao(Config conf) { @Override public Iterator> get(MemorySegment from, MemorySegment to) { + if (new MemSegComparatorNull().compare(from, to) > 0) { + return reverseIter(from, to); + } + SortedMap> dataSlice; if (from == null && to == null) { @@ -44,6 +48,11 @@ public Iterator> get(MemorySegment from, MemorySegment to) return new SSTableIterator(dataSlice.values().iterator(), controller, from, to); } + private Iterator> reverseIter(MemorySegment fromRightSide, MemorySegment toLeftSide) { + return new SSTableIterator(getMemTable().reversed().values().iterator(), + controller, fromRightSide, toLeftSide, true, new MemSegComparatorNull().reversed()); + } + @Override public Entry get(MemorySegment key) { if (key == null) { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java new file mode 100644 index 000000000..1bd2542b8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java @@ -0,0 +1,35 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.SortedMap; + +public class IteratorUtils { + public static void insertNew(SortedMap mp, + SSTablesController controller, SSTableRowInfo info, MemorySegment to) { + Entry kv = controller.getRow(info); + + if (kv == null) { + return; + } + + if (!mp.containsKey(kv.key())) { + mp.put(kv.key(), info); + return; + } + SSTableRowInfo old = mp.get(kv.key()); + + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; + SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; + + mp.put(controller.getRow(newInfo).key(), newInfo); + + // tail recursion + if (oldInfo.isReversedToIter) { + insertNew(mp, controller, controller.getPrevInfo(oldInfo, to), to); + } else { + insertNew(mp, controller, controller.getNextInfo(oldInfo, to), to); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 6d53ce9b3..d61950d09 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -14,17 +14,21 @@ public class SSTableIterator implements Iterator> { private final Iterator> memTableIterator; private final SSTablesController controller; - private final Comparator comp = new MemSegComparatorNull(); - private final SortedMap mp = new TreeMap<>(comp); + private final Comparator comp; + private final SortedMap mp; private final MemorySegment from; private final MemorySegment to; private Entry head; private Entry keeper; + private boolean isReversed = false; + public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { - memTableIterator = it; + this.memTableIterator = it; this.controller = controller; + this.comp = new MemSegComparatorNull(); + this.mp = new TreeMap<>(comp); this.from = from; this.to = to; @@ -32,33 +36,26 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } - private void insertNew(SSTableRowInfo info) { - Entry kv = controller.getRow(info); - - if (kv == null) { - return; - } - - if (!mp.containsKey(kv.key())) { - mp.put(kv.key(), info); - return; - } - SSTableRowInfo old = mp.get(kv.key()); - - SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; - SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; + public SSTableIterator(Iterator> it, SSTablesController controller, + MemorySegment from, MemorySegment to, boolean isReversed, Comparator comp) { + this.memTableIterator = it; + this.controller = controller; + this.comp = comp; + this.mp = new TreeMap<>(comp); - mp.put(controller.getRow(newInfo).key(), newInfo); + this.from = from; + this.to = to; + this.isReversed = true; - // tail recursion - insertNew(controller.getNextInfo(oldInfo, to)); + positioningIterator(); } + private void positioningIterator() { - List rawData = controller.firstGreaterKeys(from); + List rawData = controller.firstKeys(from, isReversed); for (var info : rawData) { - insertNew(info); + IteratorUtils.insertNew(mp, controller, info, to); } } @@ -73,6 +70,7 @@ public boolean hasNext() { private Entry getHead() { if (head == null && memTableIterator.hasNext()) { head = memTableIterator.next(); + // fixme ended if (comp.compare(head.key(), to) >= 0) { head = null; } @@ -138,7 +136,13 @@ private void changeState() { } private void updateMp(MemorySegment key) { - insertNew(controller.getNextInfo(mp.remove(key), to)); + if (isReversed) { + IteratorUtils.insertNew(mp, controller, + controller.getPrevInfo(mp.remove(key), to), to); + return; + } + IteratorUtils.insertNew(mp, controller, + controller.getNextInfo(mp.remove(key), to), to); } private boolean isBetween(MemorySegment who) { @@ -150,8 +154,7 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); while (!mp.isEmpty() && !isBetween(minSStablesEntry.getKey())) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + updateMp(minSStablesEntry.getKey()); minSStablesEntry = mp.firstEntry(); } return minSStablesEntry; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 309a26400..1758cb3ad 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -7,7 +7,7 @@ public class SSTableRowInfo { long rowShift; private final long valueSize; int ssTableInd; - + boolean isReversedToIter = false; public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift) { this.keyOffset = keyOffset; @@ -17,6 +17,17 @@ public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, this.ssTableInd = ssTableInd; this.rowShift = rowShift; } + public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, + long valueSize, int ssTableInd, long rowShift, boolean isReversedToIter) { + this.keyOffset = keyOffset; + this.valueOffset = valueOffset; + this.keySize = keySize; + this.valueSize = valueSize; + this.ssTableInd = ssTableInd; + this.rowShift = rowShift; + this.isReversedToIter = isReversedToIter; + } + public boolean isDeletedData() { return valueSize < 0; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 7341db9e3..06cc4b78c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -19,6 +19,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Stream; import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; @@ -77,6 +78,12 @@ private boolean greaterThen(long keyOffset, long keySize, return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; } + private boolean greaterEqThen(long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) >= 0; + } + //Gives offset for line in index file private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { long l = -1; @@ -94,19 +101,42 @@ private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { return r == getNumberOfEntries(mapped) ? -1 : r; } + private long searchKeyInFileReversed(int ind, MemorySegment mapped, MemorySegment key) { + long l = -1; + long r = getNumberOfEntries(mapped); + + while (r - l > 1) { + long mid = (l + r) / 2; + SSTableRowInfo info = createRowInfo(ind, mid); + if (greaterEqThen(info.keyOffset, info.keySize, mapped, key)) { + l = mid; + } else { + r = mid; + } + } + return l; + } + //return - List ordered form the latest created sstable to the first. - public List firstGreaterKeys(MemorySegment key) { + public List firstKeys(MemorySegment key, boolean isReversed) { List ans = new ArrayList<>(); for (int i = ssTables.size() - 1; i >= 0; i--) { long entryIndexesLine = 0; if (key != null) { - entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); + if (!isReversed) { + entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); + } else { + entryIndexesLine = searchKeyInFileReversed(i, ssTables.get(i), key); + } } if (entryIndexesLine < 0) { continue; } - ans.add(createRowInfo(i, entryIndexesLine)); + SSTableRowInfo row = createRowInfo(i, entryIndexesLine); + row.isReversedToIter = isReversed; + + ans.add(row); } return ans; } @@ -160,6 +190,19 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return null; } + public SSTableRowInfo getPrevInfo(SSTableRowInfo info, MemorySegment minKey) { + for (long t = info.rowShift - 1; t >= 0; t--) { + var inf = createRowInfo(info.ssTableInd, t); + + Entry row = getRow(inf); + // fixme include? + if (segComp.compare(row.key(), minKey) > 0) { + return inf; + } + } + return null; + } + private long getNumberOfEntries(MemorySegment memSeg) { return memSeg.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } diff --git a/src/test/java/ru/vk/itmo/ReverseIteratorTest.java b/src/test/java/ru/vk/itmo/ReverseIteratorTest.java new file mode 100644 index 000000000..93ffb58b7 --- /dev/null +++ b/src/test/java/ru/vk/itmo/ReverseIteratorTest.java @@ -0,0 +1,158 @@ +package ru.vk.itmo; + +import ru.vk.itmo.test.DaoFactory; + +import java.io.IOException; +import java.util.List; + +public class ReverseIteratorTest extends BaseTest { + @DaoTest(stage = 4, maxStage = 4) + void memTableCheckReverse(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + } + + @DaoTest(stage = 4, maxStage = 4) + void memTableCheckOrdinary(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + + assertSame(dao.all(), entries); + } + + @DaoTest(stage = 4, maxStage = 4) + void ssTablesCheckReverse(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + } + + @DaoTest(stage = 4, maxStage = 4) + void ssTablesCheckOrdinary(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + + assertSame(dao.all(), entries); + } + + @DaoTest(stage = 4) + void compactionReverse(Dao> dao) throws IOException { + List> entries = entries(100); + List> firstHalf = entries.subList(0, 50); + List> lastHalf = entries.subList(50, 100); + + for (Entry entry : firstHalf) { + dao.upsert(entry); + } + dao.compact(); + dao.close(); + + dao = DaoFactory.Factory.reopen(dao); + for (Entry entry : lastHalf) { + dao.upsert(entry); + } + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + dao.flush(); + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + + dao.compact(); + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + } + + + @DaoTest(stage = 4) + void compactionOrdinary(Dao> dao) throws IOException { + List> entries = entries(100); + List> firstHalf = entries.subList(0, 50); + List> lastHalf = entries.subList(50, 100); + + for (Entry entry : firstHalf) { + dao.upsert(entry); + } + dao.compact(); + dao.close(); + + dao = DaoFactory.Factory.reopen(dao); + for (Entry entry : lastHalf) { + dao.upsert(entry); + } + assertSame(dao.all(), entries); + + dao.flush(); + assertSame(dao.all(), entries); + + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.all(), entries); + + dao.compact(); + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.all(), entries); + } + + @DaoTest(stage = 1) + void testOrder(Dao> dao) { + dao.upsert(entry("b", "b")); + dao.upsert(entry("aa", "aa")); + dao.upsert(entry("", "")); + + assertSame( + dao.get("b", ""), + + entry("b", "b"), + entry("aa", "aa") + ); + } + + @DaoTest(stage = 1) + void testFullRange(Dao> dao) { + dao.upsert(entry("a", "b")); + + dao.upsert(entry("b", "b")); + dao.upsert(entry("c", "c")); + + assertSame( + dao.get("z", ""), + + entry("c", "c"), + entry("b", "b"), + entry("a", "b") + ); + } +} From ac2e9772c9ca6b15c8049e2612069b046e2f851d Mon Sep 17 00:00:00 2001 From: Jenshen30 Date: Mon, 4 Dec 2023 20:19:08 +0300 Subject: [PATCH 57/57] codeclimate/// --- .../test/kachmareugene/DaoWithCompaction.java | 4 -- .../test/kachmareugene/IteratorUtils.java | 3 + .../test/kachmareugene/SSTableIterator.java | 7 +-- .../test/kachmareugene/SSTableRowInfo.java | 5 +- .../kachmareugene/SSTablesController.java | 61 +++---------------- .../ru/vk/itmo/test/kachmareugene/Utils.java | 52 ++++++++++++++++ 6 files changed, 69 insertions(+), 63 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java index e0750d132..b18717438 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java @@ -1,11 +1,7 @@ package ru.vk.itmo.test.kachmareugene; import ru.vk.itmo.Config; -import ru.vk.itmo.Entry; - import java.io.IOException; -import java.lang.foreign.MemorySegment; -import java.util.Iterator; public class DaoWithCompaction extends InMemoryDao { public DaoWithCompaction() { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java index 1bd2542b8..9d4214c87 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java @@ -6,6 +6,9 @@ import java.util.SortedMap; public class IteratorUtils { + private IteratorUtils() { + } + public static void insertNew(SortedMap mp, SSTablesController controller, SSTableRowInfo info, MemorySegment to) { Entry kv = controller.getRow(info); diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index d61950d09..9b74a6207 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -20,8 +20,7 @@ public class SSTableIterator implements Iterator> { private final MemorySegment to; private Entry head; private Entry keeper; - - private boolean isReversed = false; + private boolean isReversed; public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { @@ -45,12 +44,11 @@ public SSTableIterator(Iterator> it, SSTablesController con this.from = from; this.to = to; - this.isReversed = true; + this.isReversed = isReversed; positioningIterator(); } - private void positioningIterator() { List rawData = controller.firstKeys(from, isReversed); @@ -70,7 +68,6 @@ public boolean hasNext() { private Entry getHead() { if (head == null && memTableIterator.hasNext()) { head = memTableIterator.next(); - // fixme ended if (comp.compare(head.key(), to) >= 0) { head = null; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 1758cb3ad..ee9e16301 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -7,7 +7,8 @@ public class SSTableRowInfo { long rowShift; private final long valueSize; int ssTableInd; - boolean isReversedToIter = false; + boolean isReversedToIter; + public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift) { this.keyOffset = keyOffset; @@ -17,6 +18,7 @@ public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, this.ssTableInd = ssTableInd; this.rowShift = rowShift; } + public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift, boolean isReversedToIter) { this.keyOffset = keyOffset; @@ -28,7 +30,6 @@ public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, this.isReversedToIter = isReversedToIter; } - public boolean isDeletedData() { return valueSize < 0; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 06cc4b78c..00be25143 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -19,7 +19,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.function.Function; import java.util.stream.Stream; import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; @@ -72,51 +71,6 @@ private List openFiles(Path dir, String fileNamePref, List } } - private boolean greaterThen(long keyOffset, long keySize, - MemorySegment mapped, MemorySegment key) { - - return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; - } - - private boolean greaterEqThen(long keyOffset, long keySize, - MemorySegment mapped, MemorySegment key) { - - return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) >= 0; - } - - //Gives offset for line in index file - private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { - long l = -1; - long r = getNumberOfEntries(mapped); - - while (r - l > 1) { - long mid = (l + r) / 2; - SSTableRowInfo info = createRowInfo(ind, mid); - if (greaterThen(info.keyOffset, info.keySize, mapped, key)) { - l = mid; - } else { - r = mid; - } - } - return r == getNumberOfEntries(mapped) ? -1 : r; - } - - private long searchKeyInFileReversed(int ind, MemorySegment mapped, MemorySegment key) { - long l = -1; - long r = getNumberOfEntries(mapped); - - while (r - l > 1) { - long mid = (l + r) / 2; - SSTableRowInfo info = createRowInfo(ind, mid); - if (greaterEqThen(info.keyOffset, info.keySize, mapped, key)) { - l = mid; - } else { - r = mid; - } - } - return l; - } - //return - List ordered form the latest created sstable to the first. public List firstKeys(MemorySegment key, boolean isReversed) { List ans = new ArrayList<>(); @@ -124,10 +78,13 @@ public List firstKeys(MemorySegment key, boolean isReversed) { for (int i = ssTables.size() - 1; i >= 0; i--) { long entryIndexesLine = 0; if (key != null) { - if (!isReversed) { - entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); + if (isReversed) { + entryIndexesLine = Utils.searchKeyInFileReversed(this, i, + getNumberOfEntries(ssTables.get(i)), + ssTables.get(i), key, segComp); } else { - entryIndexesLine = searchKeyInFileReversed(i, ssTables.get(i), key); + entryIndexesLine = Utils.searchKeyInFile(this, i, + getNumberOfEntries(ssTables.get(i)), ssTables.get(i), key, segComp); } } if (entryIndexesLine < 0) { @@ -141,7 +98,7 @@ public List firstKeys(MemorySegment key, boolean isReversed) { return ans; } - private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { + public SSTableRowInfo createRowInfo(int ind, final long rowIndex) { long start = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES); long size = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES * 2); @@ -152,7 +109,8 @@ private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { public SSTableRowInfo searchInSStables(MemorySegment key) { for (int i = ssTables.size() - 1; i >= 0; i--) { - long ind = searchKeyInFile(i, ssTables.get(i), key); + long ind = Utils.searchKeyInFile(this, i, + getNumberOfEntries(ssTables.get(i)), ssTables.get(i), key, segComp); if (ind >= 0) { return createRowInfo(i, ind); } @@ -195,7 +153,6 @@ public SSTableRowInfo getPrevInfo(SSTableRowInfo info, MemorySegment minKey) { var inf = createRowInfo(info.ssTableInd, t); Entry row = getRow(inf); - // fixme include? if (segComp.compare(row.key(), minKey) > 0) { return inf; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 77886d9d3..913c8b521 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -72,4 +72,56 @@ public static MemorySegment getValueOrNull(Entry kv) { } return value; } + + private static boolean greaterThen(Comparator segComp, long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; + } + + private static boolean greaterEqThen(Comparator segComp, long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) >= 0; + } + + //Gives offset for line in index file + public static long searchKeyInFile(SSTablesController controller, + int ind, long numberOfEntries, + MemorySegment mapped, MemorySegment key, + Comparator segComp) { + long l = -1; + long r = numberOfEntries; + + while (r - l > 1) { + long mid = (l + r) / 2; + SSTableRowInfo info = controller.createRowInfo(ind, mid); + if (greaterThen(segComp, info.keyOffset, info.keySize, mapped, key)) { + l = mid; + } else { + r = mid; + } + } + return r == numberOfEntries ? -1 : r; + } + + public static long searchKeyInFileReversed(SSTablesController controller, + int ind, long numberOfEntries, + MemorySegment mapped, MemorySegment key, + Comparator segComp) { + long l = -1; + long r = numberOfEntries; + + while (r - l > 1) { + long mid = (l + r) / 2; + SSTableRowInfo info = controller.createRowInfo(ind, mid); + if (greaterEqThen(segComp, info.keyOffset, info.keySize, mapped, key)) { + l = mid; + } else { + r = mid; + } + } + return l; + } + }