From e176c7d62331fd49ad9d2a857f3c18f7b514f81e Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 27 Sep 2023 04:05:26 +0300 Subject: [PATCH 01/23] all done --- .../DaoFactoryImplementation.java | 35 ++++++++++++++ .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 47 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java create mode 100644 src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java new file mode 100644 index 000000000..31a79b1d2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java @@ -0,0 +1,35 @@ +package ru.vk.itmo.test.tyapuevdmitrij; + +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; +import ru.vk.itmo.test.DaoFactory; +import ru.vk.itmo.tyapuevdmitrij.InMemoryDao; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +@DaoFactory +public class DaoFactoryImplementation implements DaoFactory.Factory> { + + @Override + public Dao> createDao() { + return new InMemoryDao(); + } + + @Override + public String toString(MemorySegment memorySegment) { + return memorySegment == null + ? null : new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); + } + + @Override + public MemorySegment fromString(String data) { + return data == null ? null : MemorySegment.ofArray(data.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public Entry fromBaseEntry(Entry baseEntry) { + return baseEntry; + } +} \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java new file mode 100644 index 000000000..705a6fd85 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -0,0 +1,47 @@ +package ru.vk.itmo.tyapuevdmitrij; + +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; +import java.util.Iterator; +import java.util.SortedMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class InMemoryDao implements Dao> { + private final Comparator memorySegmentComparator = (segment1, segment2) -> { + if (segment1.byteSize() == segment2.byteSize()) { + long offset = segment1.mismatch(segment2); + if (offset == -1) { + return 0; + } else return segment1.get(ValueLayout.JAVA_BYTE, offset) - segment2.get(ValueLayout.JAVA_BYTE, offset); + } + return Long.compare(segment1.byteSize(), segment2.byteSize()); + }; + private final SortedMap> dataMap + = new ConcurrentSkipListMap<>(memorySegmentComparator); + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return dataMap.values().iterator(); + } else if (from == null) { + return dataMap.headMap(to).values().iterator(); + } else if (to == null) { + return dataMap.tailMap(from).values().iterator(); + } + return dataMap.subMap(from, to).values().iterator(); + } + + @Override + public Entry get(MemorySegment key) { + return dataMap.get(key); + } + + @Override + public void upsert(Entry entry) { + dataMap.put(entry.key(), entry); + } +} \ No newline at end of file From 26f5bb7371cc07443595ad3df1c3547b03bc0a67 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 27 Sep 2023 04:34:24 +0300 Subject: [PATCH 02/23] fixed codeclimate --- .../vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java | 3 ++- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java index 31a79b1d2..bd3985830 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java @@ -32,4 +32,5 @@ public MemorySegment fromString(String data) { public Entry fromBaseEntry(Entry baseEntry) { return baseEntry; } -} \ No newline at end of file +} + diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 705a6fd85..954ab00ad 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -44,4 +44,6 @@ public Entry get(MemorySegment key) { public void upsert(Entry entry) { dataMap.put(entry.key(), entry); } -} \ No newline at end of file +} + + From 5a89728e6427c6e24482082e9025c7276cbd7813 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 27 Sep 2023 04:45:27 +0300 Subject: [PATCH 03/23] fixes --- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 954ab00ad..a32b8f9ee 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -7,7 +7,7 @@ import java.lang.foreign.ValueLayout; import java.util.Comparator; import java.util.Iterator; -import java.util.SortedMap; +import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDao implements Dao> { @@ -20,7 +20,7 @@ public class InMemoryDao implements Dao> { } return Long.compare(segment1.byteSize(), segment2.byteSize()); }; - private final SortedMap> dataMap + private final ConcurrentNavigableMap> dataMap = new ConcurrentSkipListMap<>(memorySegmentComparator); @Override From 269b43f5d411551ada8d6a3b03a5ed35b32c86c9 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 27 Sep 2023 20:49:43 +0300 Subject: [PATCH 04/23] fix new test --- .../ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index a32b8f9ee..b0b771962 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -12,13 +12,17 @@ public class InMemoryDao implements Dao> { private final Comparator memorySegmentComparator = (segment1, segment2) -> { - if (segment1.byteSize() == segment2.byteSize()) { - long offset = segment1.mismatch(segment2); - if (offset == -1) { - return 0; - } else return segment1.get(ValueLayout.JAVA_BYTE, offset) - segment2.get(ValueLayout.JAVA_BYTE, offset); + long offset = segment1.mismatch(segment2); + if (offset == -1) { + return 0; } - return Long.compare(segment1.byteSize(), segment2.byteSize()); + if (offset == segment1.byteSize()) { + return -1; + } + if (offset == segment2.byteSize()) { + return 1; + } + return segment1.get(ValueLayout.JAVA_BYTE, offset) - segment2.get(ValueLayout.JAVA_BYTE, offset); }; private final ConcurrentNavigableMap> dataMap = new ConcurrentSkipListMap<>(memorySegmentComparator); From c162794248805714bb81d00483b29670ecfef16c Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 4 Oct 2023 07:24:40 +0300 Subject: [PATCH 05/23] Homework 2 done --- .../DaoFactoryImplementation.java | 8 +- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 113 ++++++++++++++++-- 2 files changed, 113 insertions(+), 8 deletions(-) diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java index bd3985830..b4f89905d 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java @@ -1,5 +1,6 @@ package ru.vk.itmo.test.tyapuevdmitrij; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; import ru.vk.itmo.test.DaoFactory; @@ -9,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory +@DaoFactory(stage = 2) public class DaoFactoryImplementation implements DaoFactory.Factory> { @Override @@ -17,6 +18,11 @@ public Dao> createDao() { return new InMemoryDao(); } + @Override + public Dao> createDao(Config config) { + return new InMemoryDao(config); + } + @Override public String toString(MemorySegment memorySegment) { return memorySegment == null diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index b0b771962..18d6807e8 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -1,12 +1,23 @@ package ru.vk.itmo.tyapuevdmitrij; +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Comparator; +import java.util.EnumSet; import java.util.Iterator; +import java.util.Map; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -24,30 +35,118 @@ public class InMemoryDao implements Dao> { } return segment1.get(ValueLayout.JAVA_BYTE, offset) - segment2.get(ValueLayout.JAVA_BYTE, offset); }; - private final ConcurrentNavigableMap> dataMap + private final ConcurrentNavigableMap> memTable = new ConcurrentSkipListMap<>(memorySegmentComparator); + private final Path ssTablePath; + private final MemorySegment ssTable; + + public InMemoryDao() { + ssTablePath = null; + ssTable = null; + } + + public InMemoryDao(Config config) { + ssTablePath = config.basePath().resolve("ssTable"); + ssTable = getReadBufferFromSsTable(); + } @Override public Iterator> get(MemorySegment from, MemorySegment to) { if (from == null && to == null) { - return dataMap.values().iterator(); + return memTable.values().iterator(); } else if (from == null) { - return dataMap.headMap(to).values().iterator(); + return memTable.headMap(to).values().iterator(); } else if (to == null) { - return dataMap.tailMap(from).values().iterator(); + return memTable.tailMap(from).values().iterator(); } - return dataMap.subMap(from, to).values().iterator(); + return memTable.subMap(from, to).values().iterator(); } @Override public Entry get(MemorySegment key) { - return dataMap.get(key); + if (memTable.containsKey(key) || ssTable == null) { + return memTable.get(key); + } + putSsTableDataToMemTable(); + return memTable.get(key); } @Override public void upsert(Entry entry) { - dataMap.put(entry.key(), entry); + memTable.put(entry.key(), entry); + } + + @Override + public void flush() throws IOException { + throw new UnsupportedEncodingException(""); + } + + @Override + public void close() throws IOException { + MemorySegment buffer = getWriteBufferToSsTable(); + writeMemTableDataToFile(buffer); + } + + public long getSsTableDataByteSize() { + long ssTableDataByteSize = 0; + for (Map.Entry> entry : memTable.entrySet()) { + ssTableDataByteSize += entry.getKey().byteSize(); + ssTableDataByteSize += entry.getValue().value().byteSize(); + } + return ssTableDataByteSize + memTable.size() * Long.BYTES * 2L; + } + + public MemorySegment getWriteBufferToSsTable() throws IOException { + return FileChannel.open(ssTablePath, EnumSet.of( + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING)) + .map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), Arena.ofAuto()); + } + + public void writeMemTableDataToFile(MemorySegment buffer) { + long offset = 0; + for (Entry entry : memTable.values()) { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); + offset += Long.BYTES; + MemorySegment.copy(entry.key(), 0, buffer, offset, entry.key().byteSize()); + offset += entry.key().byteSize(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); + offset += Long.BYTES; + MemorySegment.copy(entry.value(), 0, buffer, offset, entry.value().byteSize()); + offset += entry.value().byteSize(); + } + } + + public MemorySegment getReadBufferFromSsTable() { + MemorySegment buffer; + try { + buffer = FileChannel.open(ssTablePath, StandardOpenOption.READ) + .map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); + } catch (IOException e) { + buffer = null; + } + return buffer; + } + + public void putSsTableDataToMemTable() { + long offset = 0; + MemorySegment keyFromSsTable; + MemorySegment valueFromSsTable; + while (offset < ssTable.byteSize()) { + long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + offset += Long.BYTES; + keyFromSsTable = ssTable.asSlice(offset, keyByteSize); + offset += keyByteSize; + long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + offset += Long.BYTES; + valueFromSsTable = ssTable.asSlice(offset, valueByteSize); + offset += valueByteSize; + upsert(new BaseEntry<>(keyFromSsTable, valueFromSsTable)); + } } } + From ca1356bcebf2dd822ff908c72f8adf6a96123ccb Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 4 Oct 2023 08:10:31 +0300 Subject: [PATCH 06/23] fixes --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 18d6807e8..925d7154a 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -97,12 +97,15 @@ public long getSsTableDataByteSize() { } public MemorySegment getWriteBufferToSsTable() throws IOException { - return FileChannel.open(ssTablePath, EnumSet.of( - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING)) - .map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), Arena.ofAuto()); + MemorySegment buffer; + try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of( + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING))) { + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), Arena.ofAuto()); + } + return buffer; } public void writeMemTableDataToFile(MemorySegment buffer) { @@ -119,11 +122,12 @@ public void writeMemTableDataToFile(MemorySegment buffer) { } } - public MemorySegment getReadBufferFromSsTable() { + public final MemorySegment getReadBufferFromSsTable() { MemorySegment buffer; + FileChannel channel; try { - buffer = FileChannel.open(ssTablePath, StandardOpenOption.READ) - .map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); + channel = FileChannel.open(ssTablePath, StandardOpenOption.READ); + buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); } catch (IOException e) { buffer = null; } From 62806341f38c7adfbe736776b3de186ee8e60735 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Wed, 4 Oct 2023 08:46:39 +0300 Subject: [PATCH 07/23] fix codeclimate --- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 925d7154a..609db2602 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -124,10 +124,8 @@ public void writeMemTableDataToFile(MemorySegment buffer) { public final MemorySegment getReadBufferFromSsTable() { MemorySegment buffer; - FileChannel channel; - try { - channel = FileChannel.open(ssTablePath, StandardOpenOption.READ); - buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); + try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { + buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); } catch (IOException e) { buffer = null; } From d4d384afbf46fb3e3aae35823c1cdb81ffcada58 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 9 Oct 2023 01:57:11 +0300 Subject: [PATCH 08/23] fix all --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index d20f7cc4b..d447f1909 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -6,7 +6,6 @@ import ru.vk.itmo.Entry; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -39,6 +38,7 @@ public class InMemoryDao implements Dao> { = new ConcurrentSkipListMap<>(memorySegmentComparator); private final Path ssTablePath; private final MemorySegment ssTable; + private final String ssTableFileName = "ssTable"; public InMemoryDao() { ssTablePath = null; @@ -46,7 +46,7 @@ public InMemoryDao() { } public InMemoryDao(Config config) { - ssTablePath = config.basePath().resolve("ssTable"); + ssTablePath = config.basePath().resolve(ssTableFileName); ssTable = getReadBufferFromSsTable(); } @@ -64,11 +64,11 @@ public Iterator> get(MemorySegment from, MemorySegment to) @Override public Entry get(MemorySegment key) { - if (memTable.containsKey(key) || ssTable == null) { - return memTable.get(key); + Entry value = memTable.get(key); + if (value != null || ssTable == null) { + return value; } - putSsTableDataToMemTable(); - return memTable.get(key); + return getSsTableDataByKey(key); } @Override @@ -78,7 +78,7 @@ public void upsert(Entry entry) { @Override public void flush() throws IOException { - throw new UnsupportedEncodingException(""); + throw new UnsupportedOperationException(""); } @Override @@ -103,7 +103,7 @@ public MemorySegment getWriteBufferToSsTable() throws IOException { StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) { - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), Arena.ofAuto()); + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), Arena.ofAuto()); } return buffer; } @@ -125,27 +125,28 @@ public void writeMemTableDataToFile(MemorySegment buffer) { public final MemorySegment getReadBufferFromSsTable() { MemorySegment buffer; try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { - buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); + buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); } catch (IOException e) { buffer = null; } return buffer; } - public void putSsTableDataToMemTable() { + public Entry getSsTableDataByKey(MemorySegment key) { long offset = 0; - MemorySegment keyFromSsTable; MemorySegment valueFromSsTable; while (offset < ssTable.byteSize()) { long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; - keyFromSsTable = ssTable.asSlice(offset, keyByteSize); - offset += keyByteSize; + long keysMismatch = MemorySegment.mismatch(ssTable, offset, offset += keyByteSize, key, 0, key.byteSize()); long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; valueFromSsTable = ssTable.asSlice(offset, valueByteSize); + if (keysMismatch == -1) { + return new BaseEntry<>(key, valueFromSsTable); + } offset += valueByteSize; - upsert(new BaseEntry<>(keyFromSsTable, valueFromSsTable)); } + return null; } } From 43569ac1e33106554c78ebb1c8cc73ca42708996 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 9 Oct 2023 02:13:46 +0300 Subject: [PATCH 09/23] fix codeclimate --- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index d447f1909..0714eab45 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -38,7 +38,7 @@ public class InMemoryDao implements Dao> { = new ConcurrentSkipListMap<>(memorySegmentComparator); private final Path ssTablePath; private final MemorySegment ssTable; - private final String ssTableFileName = "ssTable"; + private static final String ssTableFileName = "ssTable"; public InMemoryDao() { ssTablePath = null; @@ -138,7 +138,9 @@ public Entry getSsTableDataByKey(MemorySegment key) { while (offset < ssTable.byteSize()) { long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; - long keysMismatch = MemorySegment.mismatch(ssTable, offset, offset += keyByteSize, key, 0, key.byteSize()); + long keysMismatch; + keysMismatch = MemorySegment.mismatch(ssTable, offset, offset + keyByteSize, key, 0, key.byteSize()); + offset += keyByteSize; long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); offset += Long.BYTES; valueFromSsTable = ssTable.asSlice(offset, valueByteSize); From 89533ca002d21c58805e79c1db47bd65a376161f Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 9 Oct 2023 02:25:28 +0300 Subject: [PATCH 10/23] last fix codeclimate --- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 0714eab45..05c8a0d1b 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -38,7 +38,7 @@ public class InMemoryDao implements Dao> { = new ConcurrentSkipListMap<>(memorySegmentComparator); private final Path ssTablePath; private final MemorySegment ssTable; - private static final String ssTableFileName = "ssTable"; + private static final String ssTableFileName = "SS_TABLE"; public InMemoryDao() { ssTablePath = null; From 0fca460ccdafa397f79f737c01b049792e155678 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 9 Oct 2023 02:28:59 +0300 Subject: [PATCH 11/23] last fix codeclimate --- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 05c8a0d1b..d50c444fe 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -38,7 +38,7 @@ public class InMemoryDao implements Dao> { = new ConcurrentSkipListMap<>(memorySegmentComparator); private final Path ssTablePath; private final MemorySegment ssTable; - private static final String ssTableFileName = "SS_TABLE"; + private static final String SS_TABLE_FILE_NAME = "ssTable"; public InMemoryDao() { ssTablePath = null; @@ -46,7 +46,7 @@ public InMemoryDao() { } public InMemoryDao(Config config) { - ssTablePath = config.basePath().resolve(ssTableFileName); + ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME); ssTable = getReadBufferFromSsTable(); } From ef2c3b03ff9d7cb477b30ab9755dcd87dcc2c4d1 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 9 Oct 2023 02:41:03 +0300 Subject: [PATCH 12/23] =?UTF-8?q?=D1=87=D1=83=D0=B6=D0=BE=D0=B9=20=D1=82?= =?UTF-8?q?=D0=B5=D1=81=D1=82=20=D0=BF=D0=BE=D0=B2=D0=B0=D0=BB=D0=B8=D0=BB?= =?UTF-8?q?=D1=81=D1=8F=20=D0=BF=D0=BE=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0?= =?UTF-8?q?=D1=83=D1=82=D1=83,=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7?= =?UTF-8?q?=D0=B8=D0=BB=20=D0=BA=D0=BE=D0=B4=20=D0=B1=D0=B5=D0=B7=20=D0=B8?= =?UTF-8?q?=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index d50c444fe..8fa8b1ae3 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -149,6 +149,6 @@ public Entry getSsTableDataByKey(MemorySegment key) { } offset += valueByteSize; } - return null; + return null; } } From 3899bc4baef582d4d773bdee21151e0ecae2abcd Mon Sep 17 00:00:00 2001 From: dmitrij Date: Fri, 27 Oct 2023 05:18:55 +0300 Subject: [PATCH 13/23] Homework 4 done --- .../DaoFactoryImplementation.java | 2 +- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 376 ++++++++++++++++-- .../vk/itmo/tyapuevdmitrij/MergeIterator.java | 138 +++++++ 3 files changed, 472 insertions(+), 44 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java index b4f89905d..92ea67d93 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java @@ -10,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class DaoFactoryImplementation implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 8fa8b1ae3..4a2f60f35 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -5,6 +5,7 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import java.io.File; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -13,15 +14,20 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.EnumSet; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDao implements Dao> { - private final Comparator memorySegmentComparator = (segment1, segment2) -> { + + private static final Comparator memorySegmentComparator = (segment1, segment2) -> { long offset = segment1.mismatch(segment2); if (offset == -1) { return 0; @@ -34,29 +40,60 @@ public class InMemoryDao implements Dao> { } return segment1.get(ValueLayout.JAVA_BYTE, offset) - segment2.get(ValueLayout.JAVA_BYTE, offset); }; - private final ConcurrentNavigableMap> memTable - = new ConcurrentSkipListMap<>(memorySegmentComparator); - private final Path ssTablePath; - private final MemorySegment ssTable; + private final ConcurrentNavigableMap> memTable = + new ConcurrentSkipListMap<>(memorySegmentComparator); + private Path ssTablePath; + private final List ssTables; private static final String SS_TABLE_FILE_NAME = "ssTable"; + private final int ssTablesQuantity; + + private final Arena readArena; + private final Arena writeArena; + private final Arena compactionWriteArena; + private final Config config; + private long ssTablesEntryQuantity; + private boolean compacted = false; public InMemoryDao() { ssTablePath = null; - ssTable = null; + ssTables = null; + ssTablesQuantity = 0; + readArena = null; + writeArena = null; + config = null; + compactionWriteArena = null; } public InMemoryDao(Config config) { - ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME); - ssTable = getReadBufferFromSsTable(); + this.config = config; + ssTablesQuantity = findSsTablesQuantity(config); + ssTables = new ArrayList<>(ssTablesQuantity); + ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); + writeArena = Arena.ofShared(); + readArena = Arena.ofShared(); + if (ssTablesQuantity != 0) { + for (int i = 0; i < ssTablesQuantity; i++) { + ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + i); + ssTables.add(getReadBufferFromSsTable(ssTablePath)); + } + } + ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); + compactionWriteArena = Arena.ofShared(); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { + return range(getMemTableIterator(from, to), from, to); + } + + private Iterator> getMemTableIterator(MemorySegment from, MemorySegment to) { if (from == null && to == null) { return memTable.values().iterator(); - } else if (from == null) { + } + if (from == null) { return memTable.headMap(to).values().iterator(); - } else if (to == null) { + } + if (to == null) { return memTable.tailMap(from).values().iterator(); } return memTable.subMap(from, to).values().iterator(); @@ -65,7 +102,10 @@ public Iterator> get(MemorySegment from, MemorySegment to) @Override public Entry get(MemorySegment key) { Entry value = memTable.get(key); - if (value != null || ssTable == null) { + if (memTable.containsKey(key) && value.value() == null) { + return null; + } + if (value != null || ssTables == null) { return value; } return getSsTableDataByKey(key); @@ -77,78 +117,328 @@ public void upsert(Entry entry) { } @Override - public void flush() throws IOException { - throw new UnsupportedOperationException(""); + public void compact() throws IOException { + if (ssTablesQuantity == 0 && memTable.isEmpty()) { + return; + } + Iterator> dataIterator = get(null, null); + MemorySegment buffer = getWriteBufferToCompaction(); + long bufferByteSize = buffer.byteSize(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, ssTablesEntryQuantity); + long[] offsets = new long[2]; + offsets[1] = bufferByteSize - Long.BYTES - ssTablesEntryQuantity * 2L * Long.BYTES; + while (dataIterator.hasNext()) { + writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); + } + compactionWriteArena.close(); + deleteOldSsTables(); + compacted = true; + } + + private void deleteOldSsTables() { + File directory = new File(config.basePath().toUri()); + if (directory.exists() && directory.isDirectory()) { + if (directory.exists() && directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { + boolean deleted = file.delete(); + if (!deleted) { + throw new RuntimeException(); + } + } + } + } + } + File[] remainingFiles = directory.listFiles(); + if (remainingFiles != null && remainingFiles.length == 1) { + File remainingFile = remainingFiles[0]; + String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; + boolean renamed = remainingFile.renameTo(new File(newFilePath)); + if (!renamed) { + throw new RuntimeException(); + } + } + + } + } + + private void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, + Entry entry, long[] offsets) { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); + offsets[1] += Long.BYTES; + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.key().byteSize()); + offsets[0] += Long.BYTES; + MemorySegment.copy(entry.key(), 0, buffer, offsets[0], entry.key().byteSize()); + offsets[0] += entry.key().byteSize(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); + offsets[1] += Long.BYTES; + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.value().byteSize()); + offsets[0] += Long.BYTES; + MemorySegment.copy(entry.value(), 0, buffer, offsets[0], entry.value().byteSize()); + offsets[0] += entry.value().byteSize(); + } + + private long getCompactionTableByteSize() { + Iterator> dataIterator = get(null, null); + long compactionTableByteSize = 0; + long countEntry = 0; + while (dataIterator.hasNext()) { + Entry entry = dataIterator.next(); + compactionTableByteSize += entry.key().byteSize(); + compactionTableByteSize += entry.value().byteSize(); + countEntry++; + } + ssTablesEntryQuantity = countEntry; + return compactionTableByteSize + countEntry * 4L * Long.BYTES + Long.BYTES; } @Override public void close() throws IOException { + if (compacted) { + return; + } + if (!readArena.scope().isAlive()) { + return; + } + readArena.close(); + if (memTable.isEmpty()) { + writeArena.close(); + return; + } MemorySegment buffer = getWriteBufferToSsTable(); writeMemTableDataToFile(buffer); + writeArena.close(); + } + + private MemorySegment getWriteBufferToCompaction() throws IOException { + ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); + MemorySegment buffer; + try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING))) { + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getCompactionTableByteSize(), compactionWriteArena); + } + return buffer; } - public long getSsTableDataByteSize() { + private long getSsTableDataByteSize() { long ssTableDataByteSize = 0; for (Map.Entry> entry : memTable.entrySet()) { ssTableDataByteSize += entry.getKey().byteSize(); - ssTableDataByteSize += entry.getValue().value().byteSize(); + if (entry.getValue().value() != null) { + ssTableDataByteSize += entry.getValue().value().byteSize(); + } } - return ssTableDataByteSize + memTable.size() * Long.BYTES * 2L; + return ssTableDataByteSize + memTable.size() * Long.BYTES * 4L + Long.BYTES; } - public MemorySegment getWriteBufferToSsTable() throws IOException { + private MemorySegment getWriteBufferToSsTable() throws IOException { MemorySegment buffer; - try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of( - StandardOpenOption.READ, + try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) { - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), Arena.ofAuto()); + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), writeArena); } return buffer; } - public void writeMemTableDataToFile(MemorySegment buffer) { + private void writeMemTableDataToFile(MemorySegment buffer) { + /* + KeyByteSize|key|ValueByteSize|Value|...|0|KeyByteSizeOffset1|.. + .|memTableSize - 1|KeyByteSizeOffsetN|memTableSize + */ long offset = 0; + long bufferByteSize = buffer.byteSize(); + long memTableSize = memTable.size(); + long writeIndexPosition = bufferByteSize - memTableSize * 2L * Long.BYTES - Long.BYTES; + //write to the end of file size of memTable + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, memTableSize); for (Entry entry : memTable.values()) { buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); + // write keyByteSizeOffsetPosition to the end of buffer + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, writeIndexPosition, offset); + writeIndexPosition += Long.BYTES; offset += Long.BYTES; MemorySegment.copy(entry.key(), 0, buffer, offset, entry.key().byteSize()); offset += entry.key().byteSize(); - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); - offset += Long.BYTES; - MemorySegment.copy(entry.value(), 0, buffer, offset, entry.value().byteSize()); - offset += entry.value().byteSize(); + // write valueByteSizeOffsetPosition to the end of buffer next to the keyByteSizeOffsetPosition + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, writeIndexPosition, offset); + writeIndexPosition += Long.BYTES; + if (entry.value() == null) { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, tombstone(offset)); + offset += Long.BYTES; + } else { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); + offset += Long.BYTES; + MemorySegment.copy(entry.value(), 0, buffer, offset, entry.value().byteSize()); + offset += entry.value().byteSize(); + } } } - public final MemorySegment getReadBufferFromSsTable() { + private MemorySegment getReadBufferFromSsTable(Path ssTablePath) { MemorySegment buffer; + boolean created = false; try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { - buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), Arena.ofAuto()); + buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), readArena); + created = true; } catch (IOException e) { buffer = null; + } finally { + if (!created) { + readArena.close(); + } } + return buffer; } - public Entry getSsTableDataByKey(MemorySegment key) { - long offset = 0; - MemorySegment valueFromSsTable; - while (offset < ssTable.byteSize()) { - long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - long keysMismatch; - keysMismatch = MemorySegment.mismatch(ssTable, offset, offset + keyByteSize, key, 0, key.byteSize()); - offset += keyByteSize; - long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - valueFromSsTable = ssTable.asSlice(offset, valueByteSize); - if (keysMismatch == -1) { - return new BaseEntry<>(key, valueFromSsTable); + private long getSsTableIndexByKey(MemorySegment ssTable, MemorySegment key) { + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + long left = 0; + long right = memTableSize - 1L; + long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; + while (left <= right) { + long mid = (left + right) >>> 1; + long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; + MemorySegment readKey = getKeyByOffset(ssTable, midOffset); + int res = memorySegmentComparator.compare(readKey, key); + if (res == 0) { + return mid; + } + if (res > 0) { + right = mid - 1; + } else { + left = mid + 1; + } + } + return left; + } + + private Entry getSsTableEntryByIndex(MemorySegment ssTable, long index) { + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + if (index > memTableSize - 1 || index < 0) { + return null; + } + long keyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES + - (memTableSize - 1L) * Long.BYTES * 2L + index * 2L * Long.BYTES; + MemorySegment readKey = getKeyByOffset(ssTable, keyIndexOffset); + MemorySegment readValue = getValueByOffset(ssTable, keyIndexOffset + Long.BYTES); + return new BaseEntry<>(readKey, readValue); + } + + private MemorySegment getKeyByOffset(MemorySegment ssTable, long offset) { + long keyByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, keyByteSizeOffset); + long keyOffset = keyByteSizeOffset + Long.BYTES; + return ssTable.asSlice(keyOffset, keyByteSize); + } + + private MemorySegment getValueByOffset(MemorySegment ssTable, long offset) { + long valueByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, valueByteSizeOffset); + if (valueByteSize < 0) { + return null; + } + long valueOffset = valueByteSizeOffset + Long.BYTES; + return ssTable.asSlice(valueOffset, valueByteSize); + } + + private Entry getSsTableDataByKey(MemorySegment key) { + /* + KeyByteSize|key|ValueByteSize|Value|...|0|KeyByteSizeOffset1|.. + .|memTableSize - 1|KeyByteSizeOffsetN|memTableSize + */ + for (int i = ssTables.size() - 1; i > -1; i--) { + MemorySegment ssTable = ssTables.get(i); + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + long left = 0; + long right = memTableSize - 1L; + long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; + while (left <= right) { + long mid = (right + left) >>> 1; + long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; + MemorySegment readKey = getKeyByOffset(ssTable, midOffset); + int res = memorySegmentComparator.compare(readKey, key); + if (res == 0) { + MemorySegment value = getValueByOffset(ssTable, midOffset + Long.BYTES); + if (value == null) { + return null; + } + return new BaseEntry<>(key, value); + } else if (res > 0) { + right = mid - 1; + } else { + left = mid + 1; + } } - offset += valueByteSize; } - return null; + return null; + } + + private int findSsTablesQuantity(Config config) { + File dir = new File(config.basePath().toUri()); + File[] files = dir.listFiles(); + if (files != null) { + long ssTablesQuantity = Arrays.stream(files) + .filter(file -> file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) + .count(); + return (int) ssTablesQuantity; + } else return 0; + } + + private Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(ssTablesQuantity + 1); + for (MemorySegment memorySegment : ssTables) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, memorySegmentComparator)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + private Iterator> iterator(MemorySegment ssTable, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : normalize(getSsTableIndexByKey(ssTable, from)); + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + long recordIndexTo = to == null ? memTableSize : normalize(getSsTableIndexByKey(ssTable, to)); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Entry entry = getSsTableEntryByIndex(ssTable, index); + index++; + return entry; + } + }; + } + + private static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); } } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java new file mode 100644 index 000000000..04f80ead5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java @@ -0,0 +1,138 @@ +package ru.vk.itmo.tyapuevdmitrij; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T peek = peek(); + this.peek = null; + return peek; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + PeekIterator peek; + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + protected boolean skip(T t) { + return false; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peek = peek(); + if (peek == null) { + throw new NoSuchElementException(); + } + T next = peek.next(); + this.peek = null; + if (peek.hasNext()) { + priorityQueue.add(peek); + } + return next; + } +} From fd682b59941b0321d1e9e664d8c8766212aa16f9 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Fri, 27 Oct 2023 19:49:14 +0300 Subject: [PATCH 14/23] fixes --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 360 ++---------------- .../vk/itmo/tyapuevdmitrij/MergeIterator.java | 38 +- .../ru/vk/itmo/tyapuevdmitrij/Storage.java | 315 +++++++++++++++ 3 files changed, 361 insertions(+), 352 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 4a2f60f35..42fac2c18 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -1,27 +1,16 @@ package ru.vk.itmo.tyapuevdmitrij; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.io.File; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; -import java.util.EnumSet; import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -42,48 +31,29 @@ public class InMemoryDao implements Dao> { }; private final ConcurrentNavigableMap> memTable = new ConcurrentSkipListMap<>(memorySegmentComparator); - private Path ssTablePath; - private final List ssTables; - private static final String SS_TABLE_FILE_NAME = "ssTable"; - private final int ssTablesQuantity; private final Arena readArena; - private final Arena writeArena; - private final Arena compactionWriteArena; - private final Config config; + + private final Path ssTablePath; private long ssTablesEntryQuantity; - private boolean compacted = false; + private boolean compacted; + private final Storage storage; public InMemoryDao() { ssTablePath = null; - ssTables = null; - ssTablesQuantity = 0; readArena = null; - writeArena = null; - config = null; - compactionWriteArena = null; + storage = null; } public InMemoryDao(Config config) { - this.config = config; - ssTablesQuantity = findSsTablesQuantity(config); - ssTables = new ArrayList<>(ssTablesQuantity); - ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); - writeArena = Arena.ofShared(); + ssTablePath = config.basePath(); readArena = Arena.ofShared(); - if (ssTablesQuantity != 0) { - for (int i = 0; i < ssTablesQuantity; i++) { - ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + i); - ssTables.add(getReadBufferFromSsTable(ssTablePath)); - } - } - ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); - compactionWriteArena = Arena.ofShared(); + storage = new Storage(ssTablePath, readArena); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return range(getMemTableIterator(from, to), from, to); + return storage.range(getMemTableIterator(from, to), from, to, memorySegmentComparator); } private Iterator> getMemTableIterator(MemorySegment from, MemorySegment to) { @@ -105,10 +75,10 @@ public Entry get(MemorySegment key) { if (memTable.containsKey(key) && value.value() == null) { return null; } - if (value != null || ssTables == null) { + if (value != null || storage.ssTables == null) { return value; } - return getSsTableDataByKey(key); + return storage.getSsTableDataByKey(key, memorySegmentComparator); } @Override @@ -118,66 +88,37 @@ public void upsert(Entry entry) { @Override public void compact() throws IOException { - if (ssTablesQuantity == 0 && memTable.isEmpty()) { + if (storage.ssTablesQuantity == 0 && memTable.isEmpty()) { return; } Iterator> dataIterator = get(null, null); - MemorySegment buffer = getWriteBufferToCompaction(); + + MemorySegment buffer = storage.getWriteBufferToCompaction(ssTablePath, getCompactionTableByteSize()); long bufferByteSize = buffer.byteSize(); buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, ssTablesEntryQuantity); long[] offsets = new long[2]; offsets[1] = bufferByteSize - Long.BYTES - ssTablesEntryQuantity * 2L * Long.BYTES; while (dataIterator.hasNext()) { - writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); + storage.writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); } - compactionWriteArena.close(); - deleteOldSsTables(); + storage.deleteOldSsTables(ssTablePath); compacted = true; } - private void deleteOldSsTables() { - File directory = new File(config.basePath().toUri()); - if (directory.exists() && directory.isDirectory()) { - if (directory.exists() && directory.isDirectory()) { - File[] files = directory.listFiles(); - if (files != null) { - for (File file : files) { - if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { - boolean deleted = file.delete(); - if (!deleted) { - throw new RuntimeException(); - } - } - } - } - } - File[] remainingFiles = directory.listFiles(); - if (remainingFiles != null && remainingFiles.length == 1) { - File remainingFile = remainingFiles[0]; - String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; - boolean renamed = remainingFile.renameTo(new File(newFilePath)); - if (!renamed) { - throw new RuntimeException(); - } - } - + @Override + public void close() throws IOException { + if (compacted) { + return; } - } + if (memTable.isEmpty()) { + return; + } + if (!readArena.scope().isAlive()) { + return; + } + readArena.close(); + storage.save(memTable.values(), ssTablePath); - private void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, - Entry entry, long[] offsets) { - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); - offsets[1] += Long.BYTES; - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.key().byteSize()); - offsets[0] += Long.BYTES; - MemorySegment.copy(entry.key(), 0, buffer, offsets[0], entry.key().byteSize()); - offsets[0] += entry.key().byteSize(); - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); - offsets[1] += Long.BYTES; - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.value().byteSize()); - offsets[0] += Long.BYTES; - MemorySegment.copy(entry.value(), 0, buffer, offsets[0], entry.value().byteSize()); - offsets[0] += entry.value().byteSize(); } private long getCompactionTableByteSize() { @@ -194,251 +135,4 @@ private long getCompactionTableByteSize() { return compactionTableByteSize + countEntry * 4L * Long.BYTES + Long.BYTES; } - @Override - public void close() throws IOException { - if (compacted) { - return; - } - if (!readArena.scope().isAlive()) { - return; - } - readArena.close(); - if (memTable.isEmpty()) { - writeArena.close(); - return; - } - MemorySegment buffer = getWriteBufferToSsTable(); - writeMemTableDataToFile(buffer); - writeArena.close(); - } - - private MemorySegment getWriteBufferToCompaction() throws IOException { - ssTablePath = config.basePath().resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); - MemorySegment buffer; - try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING))) { - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getCompactionTableByteSize(), compactionWriteArena); - } - return buffer; - } - - private long getSsTableDataByteSize() { - long ssTableDataByteSize = 0; - for (Map.Entry> entry : memTable.entrySet()) { - ssTableDataByteSize += entry.getKey().byteSize(); - if (entry.getValue().value() != null) { - ssTableDataByteSize += entry.getValue().value().byteSize(); - } - } - return ssTableDataByteSize + memTable.size() * Long.BYTES * 4L + Long.BYTES; - } - - private MemorySegment getWriteBufferToSsTable() throws IOException { - MemorySegment buffer; - try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING))) { - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, getSsTableDataByteSize(), writeArena); - } - return buffer; - } - - private void writeMemTableDataToFile(MemorySegment buffer) { - /* - KeyByteSize|key|ValueByteSize|Value|...|0|KeyByteSizeOffset1|.. - .|memTableSize - 1|KeyByteSizeOffsetN|memTableSize - */ - long offset = 0; - long bufferByteSize = buffer.byteSize(); - long memTableSize = memTable.size(); - long writeIndexPosition = bufferByteSize - memTableSize * 2L * Long.BYTES - Long.BYTES; - //write to the end of file size of memTable - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, memTableSize); - for (Entry entry : memTable.values()) { - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); - // write keyByteSizeOffsetPosition to the end of buffer - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, writeIndexPosition, offset); - writeIndexPosition += Long.BYTES; - offset += Long.BYTES; - MemorySegment.copy(entry.key(), 0, buffer, offset, entry.key().byteSize()); - offset += entry.key().byteSize(); - // write valueByteSizeOffsetPosition to the end of buffer next to the keyByteSizeOffsetPosition - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, writeIndexPosition, offset); - writeIndexPosition += Long.BYTES; - if (entry.value() == null) { - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, tombstone(offset)); - offset += Long.BYTES; - } else { - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); - offset += Long.BYTES; - MemorySegment.copy(entry.value(), 0, buffer, offset, entry.value().byteSize()); - offset += entry.value().byteSize(); - } - } - } - - private MemorySegment getReadBufferFromSsTable(Path ssTablePath) { - MemorySegment buffer; - boolean created = false; - try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { - buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), readArena); - created = true; - } catch (IOException e) { - buffer = null; - } finally { - if (!created) { - readArena.close(); - } - } - - return buffer; - } - - private long getSsTableIndexByKey(MemorySegment ssTable, MemorySegment key) { - long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); - long left = 0; - long right = memTableSize - 1L; - long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; - while (left <= right) { - long mid = (left + right) >>> 1; - long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; - MemorySegment readKey = getKeyByOffset(ssTable, midOffset); - int res = memorySegmentComparator.compare(readKey, key); - if (res == 0) { - return mid; - } - if (res > 0) { - right = mid - 1; - } else { - left = mid + 1; - } - } - return left; - } - - private Entry getSsTableEntryByIndex(MemorySegment ssTable, long index) { - long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); - if (index > memTableSize - 1 || index < 0) { - return null; - } - long keyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES - - (memTableSize - 1L) * Long.BYTES * 2L + index * 2L * Long.BYTES; - MemorySegment readKey = getKeyByOffset(ssTable, keyIndexOffset); - MemorySegment readValue = getValueByOffset(ssTable, keyIndexOffset + Long.BYTES); - return new BaseEntry<>(readKey, readValue); - } - - private MemorySegment getKeyByOffset(MemorySegment ssTable, long offset) { - long keyByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, keyByteSizeOffset); - long keyOffset = keyByteSizeOffset + Long.BYTES; - return ssTable.asSlice(keyOffset, keyByteSize); - } - - private MemorySegment getValueByOffset(MemorySegment ssTable, long offset) { - long valueByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, valueByteSizeOffset); - if (valueByteSize < 0) { - return null; - } - long valueOffset = valueByteSizeOffset + Long.BYTES; - return ssTable.asSlice(valueOffset, valueByteSize); - } - - private Entry getSsTableDataByKey(MemorySegment key) { - /* - KeyByteSize|key|ValueByteSize|Value|...|0|KeyByteSizeOffset1|.. - .|memTableSize - 1|KeyByteSizeOffsetN|memTableSize - */ - for (int i = ssTables.size() - 1; i > -1; i--) { - MemorySegment ssTable = ssTables.get(i); - long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); - long left = 0; - long right = memTableSize - 1L; - long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; - while (left <= right) { - long mid = (right + left) >>> 1; - long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; - MemorySegment readKey = getKeyByOffset(ssTable, midOffset); - int res = memorySegmentComparator.compare(readKey, key); - if (res == 0) { - MemorySegment value = getValueByOffset(ssTable, midOffset + Long.BYTES); - if (value == null) { - return null; - } - return new BaseEntry<>(key, value); - } else if (res > 0) { - right = mid - 1; - } else { - left = mid + 1; - } - } - } - return null; - } - - private int findSsTablesQuantity(Config config) { - File dir = new File(config.basePath().toUri()); - File[] files = dir.listFiles(); - if (files != null) { - long ssTablesQuantity = Arrays.stream(files) - .filter(file -> file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) - .count(); - return (int) ssTablesQuantity; - } else return 0; - } - - private Iterator> range( - Iterator> firstIterator, - MemorySegment from, - MemorySegment to) { - List>> iterators = new ArrayList<>(ssTablesQuantity + 1); - for (MemorySegment memorySegment : ssTables) { - iterators.add(iterator(memorySegment, from, to)); - } - iterators.add(firstIterator); - - return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, memorySegmentComparator)) { - @Override - protected boolean skip(Entry memorySegmentEntry) { - return memorySegmentEntry.value() == null; - } - }; - } - - private Iterator> iterator(MemorySegment ssTable, MemorySegment from, MemorySegment to) { - long recordIndexFrom = from == null ? 0 : normalize(getSsTableIndexByKey(ssTable, from)); - long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); - long recordIndexTo = to == null ? memTableSize : normalize(getSsTableIndexByKey(ssTable, to)); - - return new Iterator<>() { - long index = recordIndexFrom; - - @Override - public boolean hasNext() { - return index < recordIndexTo; - } - - @Override - public Entry next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - Entry entry = getSsTableEntryByIndex(ssTable, index); - index++; - return entry; - } - }; - } - - private static long tombstone(long offset) { - return 1L << 63 | offset; - } - - private static long normalize(long value) { - return value & ~(1L << 63); - } } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java index 04f80ead5..212cdd34e 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java @@ -51,7 +51,7 @@ private T peek() { } } - PeekIterator peek; + PeekIterator tableIterator; public MergeIterator(Collection> iterators, Comparator comparator) { this.comparator = comparator; @@ -70,9 +70,9 @@ public MergeIterator(Collection> iterators, Comparator comparator } private PeekIterator peek() { - while (peek == null) { - peek = priorityQueue.poll(); - if (peek == null) { + while (tableIterator == null) { + tableIterator = priorityQueue.poll(); + if (tableIterator == null) { return null; } @@ -82,7 +82,7 @@ private PeekIterator peek() { break; } - int compare = comparator.compare(peek.peek(), next.peek()); + int compare = comparator.compare(tableIterator.peek(), next.peek()); if (compare == 0) { PeekIterator poll = priorityQueue.poll(); if (poll != null) { @@ -96,21 +96,21 @@ private PeekIterator peek() { } } - if (peek.peek() == null) { - peek = null; + if (tableIterator.peek() == null) { + tableIterator = null; continue; } - if (skip(peek.peek())) { - peek.next(); - if (peek.hasNext()) { - priorityQueue.add(peek); + if (skip(tableIterator.peek())) { + tableIterator.next(); + if (tableIterator.hasNext()) { + priorityQueue.add(tableIterator); } - peek = null; + tableIterator = null; } } - return peek; + return tableIterator; } protected boolean skip(T t) { @@ -124,14 +124,14 @@ public boolean hasNext() { @Override public T next() { - PeekIterator peek = peek(); - if (peek == null) { + PeekIterator entryIterator = peek(); + if (entryIterator == null) { throw new NoSuchElementException(); } - T next = peek.next(); - this.peek = null; - if (peek.hasNext()) { - priorityQueue.add(peek); + T next = entryIterator.next(); + this.tableIterator = null; + if (entryIterator.hasNext()) { + priorityQueue.add(entryIterator); } return next; } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java new file mode 100644 index 000000000..d25c6d97f --- /dev/null +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java @@ -0,0 +1,315 @@ +package ru.vk.itmo.tyapuevdmitrij; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.File; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class Storage { + protected List ssTables; + private long memTableEntriesSize; + private static final String SS_TABLE_FILE_NAME = "ssTable"; + protected int ssTablesQuantity; + + public Storage(Path ssTablePath, Arena readArena) { + ssTablesQuantity = findSsTablesQuantity(ssTablePath); + ssTables = new ArrayList<>(ssTablesQuantity); + if (ssTablesQuantity != 0) { + for (int i = 0; i < ssTablesQuantity; i++) { + Path path = ssTablePath.resolve(SS_TABLE_FILE_NAME + i); + ssTables.add(getReadBufferFromSsTable(path, readArena)); + } + } + } + + private long getSsTableDataByteSize(Iterable> memTableEntries) { + long ssTableDataByteSize = 0; + long entriesCount = 0; + for (Entry entry : memTableEntries) { + ssTableDataByteSize += entry.key().byteSize(); + if (entry.value() != null) { + ssTableDataByteSize += entry.value().byteSize(); + } + entriesCount++; + } + memTableEntriesSize = entriesCount; + return ssTableDataByteSize + entriesCount * Long.BYTES * 4L + Long.BYTES; + } + + public void deleteOldSsTables(Path ssTablePath) { + File directory = new File(ssTablePath.toUri()); + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { + file.delete(); + } + } + } + if (files != null) { + File remainingFile = files[0]; + String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; + remainingFile.renameTo(new File(newFilePath)); + } + + } + + public MemorySegment getWriteBufferToCompaction(Path ssTablePath, + Long compactionTableByteSize) throws IOException { + ssTablePath = ssTablePath.resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); + MemorySegment buffer; + Arena compactionWriteArena = Arena.ofConfined(); + try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING))) { + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, compactionTableByteSize, compactionWriteArena); + } + return buffer; + } + + public void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, + Entry entry, long[] offsets) { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); + offsets[1] += Long.BYTES; + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.key().byteSize()); + offsets[0] += Long.BYTES; + MemorySegment.copy(entry.key(), 0, buffer, offsets[0], entry.key().byteSize()); + offsets[0] += entry.key().byteSize(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); + offsets[1] += Long.BYTES; + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.value().byteSize()); + offsets[0] += Long.BYTES; + MemorySegment.copy(entry.value(), 0, buffer, offsets[0], entry.value().byteSize()); + offsets[0] += entry.value().byteSize(); + } + + private MemorySegment getWriteBufferToSsTable(Iterable> memTableEntries, Path ssTablePath + ) throws IOException { + MemorySegment buffer; + Arena writeArena = Arena.ofConfined(); + try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING))) { + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, + getSsTableDataByteSize(memTableEntries), writeArena); + } + return buffer; + } + + public void save(Iterable> memTableEntries, Path ssTablePath) throws IOException { + MemorySegment buffer = getWriteBufferToSsTable(memTableEntries, + ssTablePath.resolve(SS_TABLE_FILE_NAME + ssTablesQuantity)); + writeMemTableDataToFile(buffer, memTableEntries); + } + + private void writeMemTableDataToFile(MemorySegment buffer, Iterable> memTableEntries) { + long offset = 0; + long bufferByteSize = buffer.byteSize(); + long writeIndexPosition = bufferByteSize - memTableEntriesSize * 2L * Long.BYTES - Long.BYTES; + //write to the end of file size of memTable + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, memTableEntriesSize); + for (Entry entry : memTableEntries) { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); + // write keyByteSizeOffsetPosition to the end of buffer + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, writeIndexPosition, offset); + writeIndexPosition += Long.BYTES; + offset += Long.BYTES; + MemorySegment.copy(entry.key(), 0, buffer, offset, entry.key().byteSize()); + offset += entry.key().byteSize(); + // write valueByteSizeOffsetPosition to the end of buffer next to the keyByteSizeOffsetPosition + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, writeIndexPosition, offset); + writeIndexPosition += Long.BYTES; + if (entry.value() == null) { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, tombstone(offset)); + offset += Long.BYTES; + } else { + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); + offset += Long.BYTES; + MemorySegment.copy(entry.value(), 0, buffer, offset, entry.value().byteSize()); + offset += entry.value().byteSize(); + } + } + } + + private static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) { + MemorySegment buffer; + boolean created = false; + try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { + buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), readArena); + created = true; + } catch (IOException e) { + buffer = null; + } finally { + if (!created) { + readArena.close(); + } + } + return buffer; + } + + private long getSsTableIndexByKey(MemorySegment ssTable, MemorySegment key, + Comparator memorySegmentComparator) { + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + long left = 0; + long right = memTableSize - 1L; + long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; + while (left <= right) { + long mid = (left + right) >>> 1; + long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; + MemorySegment readKey = getKeyByOffset(ssTable, midOffset); + int res = memorySegmentComparator.compare(readKey, key); + if (res == 0) { + return mid; + } + if (res > 0) { + right = mid - 1; + } else { + left = mid + 1; + } + } + return left; + } + + private Entry getSsTableEntryByIndex(MemorySegment ssTable, long index) { + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + if (index > memTableSize - 1 || index < 0) { + return null; + } + long keyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES + - (memTableSize - 1L) * Long.BYTES * 2L + index * 2L * Long.BYTES; + MemorySegment readKey = getKeyByOffset(ssTable, keyIndexOffset); + MemorySegment readValue = getValueByOffset(ssTable, keyIndexOffset + Long.BYTES); + return new BaseEntry<>(readKey, readValue); + } + + private MemorySegment getKeyByOffset(MemorySegment ssTable, long offset) { + long keyByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + long keyByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, keyByteSizeOffset); + long keyOffset = keyByteSizeOffset + Long.BYTES; + return ssTable.asSlice(keyOffset, keyByteSize); + } + + private MemorySegment getValueByOffset(MemorySegment ssTable, long offset) { + long valueByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, valueByteSizeOffset); + if (valueByteSize < 0) { + return null; + } + long valueOffset = valueByteSizeOffset + Long.BYTES; + return ssTable.asSlice(valueOffset, valueByteSize); + } + + public Entry getSsTableDataByKey(MemorySegment key, + Comparator memorySegmentComparator) { + for (int i = ssTables.size() - 1; i > -1; i--) { + MemorySegment ssTable = ssTables.get(i); + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + long left = 0; + long right = memTableSize - 1L; + long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; + while (left <= right) { + long mid = (right + left) >>> 1; + long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; + MemorySegment readKey = getKeyByOffset(ssTable, midOffset); + int res = memorySegmentComparator.compare(readKey, key); + if (res == 0) { + MemorySegment value = getValueByOffset(ssTable, midOffset + Long.BYTES); + if (value == null) { + return null; + } + return new BaseEntry<>(key, value); + } else if (res > 0) { + right = mid - 1; + } else { + left = mid + 1; + } + } + } + return null; + } + + private int findSsTablesQuantity(Path ssTablePath) { + File dir = new File(ssTablePath.toUri()); + File[] files = dir.listFiles(); + if (files != null) { + long ssTablesQuantity = Arrays.stream(files) + .filter(file -> file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) + .count(); + return (int) ssTablesQuantity; + } else return 0; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to, Comparator memorySegmentComparator) { + List>> iterators = new ArrayList<>(ssTablesQuantity + 1); + for (MemorySegment memorySegment : ssTables) { + iterators.add(iterator(memorySegment, from, to, memorySegmentComparator)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, memorySegmentComparator)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + private Iterator> iterator(MemorySegment ssTable, MemorySegment from, MemorySegment to, + Comparator memorySegmentComparator) { + long recordIndexFrom = from == null ? 0 : normalize(getSsTableIndexByKey(ssTable, + from, + memorySegmentComparator)); + long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); + long recordIndexTo = to == null ? memTableSize : normalize(getSsTableIndexByKey(ssTable, + to, + memorySegmentComparator)); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Entry entry = getSsTableEntryByIndex(ssTable, index); + index++; + return entry; + } + }; + } + + private static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); + } + +} From 52230fd1e6b9e474691abab98dc02bb6f5d970c8 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Fri, 27 Oct 2023 21:33:20 +0300 Subject: [PATCH 15/23] several fixes for codeclimate --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 9 ++- .../vk/itmo/tyapuevdmitrij/MergeIterator.java | 58 ++++++++--------- .../ru/vk/itmo/tyapuevdmitrij/Storage.java | 63 ++++++++----------- 3 files changed, 62 insertions(+), 68 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 42fac2c18..6586d3baf 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -92,8 +92,7 @@ public void compact() throws IOException { return; } Iterator> dataIterator = get(null, null); - - MemorySegment buffer = storage.getWriteBufferToCompaction(ssTablePath, getCompactionTableByteSize()); + MemorySegment buffer = storage.getWriteBufferToSsTable(getCompactionTableByteSize(), ssTablePath); long bufferByteSize = buffer.byteSize(); buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, ssTablesEntryQuantity); long[] offsets = new long[2]; @@ -101,7 +100,11 @@ public void compact() throws IOException { while (dataIterator.hasNext()) { storage.writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); } - storage.deleteOldSsTables(ssTablePath); + try { + storage.deleteOldSsTables(ssTablePath); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } compacted = true; } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java index 212cdd34e..eb1e295e0 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java @@ -10,12 +10,13 @@ public class MergeIterator implements Iterator { private final PriorityQueue> priorityQueue; private final Comparator comparator; + PeekIterator tableIterator; private static class PeekIterator implements Iterator { public final int id; private final Iterator delegate; - private T peek; + private T memorySegmentsEntry; private PeekIterator(int id, Iterator delegate) { this.id = id; @@ -24,7 +25,7 @@ private PeekIterator(int id, Iterator delegate) { @Override public boolean hasNext() { - if (peek == null) { + if (memorySegmentsEntry == null) { return delegate.hasNext(); } return true; @@ -36,23 +37,21 @@ public T next() { throw new NoSuchElementException(); } T peek = peek(); - this.peek = null; + this.memorySegmentsEntry = null; return peek; } private T peek() { - if (peek == null) { + if (memorySegmentsEntry == null) { if (!delegate.hasNext()) { return null; } - peek = delegate.next(); + memorySegmentsEntry = delegate.next(); } - return peek; + return memorySegmentsEntry; } } - PeekIterator tableIterator; - public MergeIterator(Collection> iterators, Comparator comparator) { this.comparator = comparator; Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); @@ -75,27 +74,7 @@ private PeekIterator peek() { if (tableIterator == null) { return null; } - - while (true) { - PeekIterator next = priorityQueue.peek(); - if (next == null) { - break; - } - - int compare = comparator.compare(tableIterator.peek(), next.peek()); - if (compare == 0) { - PeekIterator poll = priorityQueue.poll(); - if (poll != null) { - poll.next(); - if (poll.hasNext()) { - priorityQueue.add(poll); - } - } - } else { - break; - } - } - + peekFromPriorityQueue(); if (tableIterator.peek() == null) { tableIterator = null; continue; @@ -113,6 +92,27 @@ private PeekIterator peek() { return tableIterator; } + private void peekFromPriorityQueue() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + int compare = comparator.compare(tableIterator.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + } + protected boolean skip(T t) { return false; } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java index d25c6d97f..ac13ff922 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java @@ -51,36 +51,25 @@ private long getSsTableDataByteSize(Iterable> memTableEntri return ssTableDataByteSize + entriesCount * Long.BYTES * 4L + Long.BYTES; } - public void deleteOldSsTables(Path ssTablePath) { + public void deleteOldSsTables(Path ssTablePath) throws NoSuchFieldException { File directory = new File(ssTablePath.toUri()); File[] files = directory.listFiles(); - if (files != null) { - for (File file : files) { - if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { - file.delete(); - } - } + if (files == null) { + throw new NoSuchFieldException(); } - if (files != null) { - File remainingFile = files[0]; - String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; - remainingFile.renameTo(new File(newFilePath)); + boolean deleted = false; + for (File file : files) { + if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { + deleted = file.delete(); + } } - - } - - public MemorySegment getWriteBufferToCompaction(Path ssTablePath, - Long compactionTableByteSize) throws IOException { - ssTablePath = ssTablePath.resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); - MemorySegment buffer; - Arena compactionWriteArena = Arena.ofConfined(); - try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING))) { - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, compactionTableByteSize, compactionWriteArena); + boolean renamed; + File remainingFile = files[0]; + String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; + renamed = remainingFile.renameTo(new File(newFilePath)); + if (!deleted && !renamed) { + throw new SecurityException(); } - return buffer; } public void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, @@ -99,23 +88,24 @@ public void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, offsets[0] += entry.value().byteSize(); } - private MemorySegment getWriteBufferToSsTable(Iterable> memTableEntries, Path ssTablePath + public MemorySegment getWriteBufferToSsTable(Long writeBytes, Path ssTablePath ) throws IOException { MemorySegment buffer; + Path path = ssTablePath.resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); Arena writeArena = Arena.ofConfined(); - try (FileChannel channel = FileChannel.open(ssTablePath, EnumSet.of(StandardOpenOption.READ, + try (FileChannel channel = FileChannel.open(path, EnumSet.of(StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) { buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, - getSsTableDataByteSize(memTableEntries), writeArena); + writeBytes, writeArena); } return buffer; } public void save(Iterable> memTableEntries, Path ssTablePath) throws IOException { - MemorySegment buffer = getWriteBufferToSsTable(memTableEntries, - ssTablePath.resolve(SS_TABLE_FILE_NAME + ssTablesQuantity)); + MemorySegment buffer = getWriteBufferToSsTable(getSsTableDataByteSize(memTableEntries), + ssTablePath); writeMemTableDataToFile(buffer, memTableEntries); } @@ -248,12 +238,13 @@ public Entry getSsTableDataByKey(MemorySegment key, private int findSsTablesQuantity(Path ssTablePath) { File dir = new File(ssTablePath.toUri()); File[] files = dir.listFiles(); - if (files != null) { - long ssTablesQuantity = Arrays.stream(files) - .filter(file -> file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) - .count(); - return (int) ssTablesQuantity; - } else return 0; + if (files == null) { + return 0; + } + long countSsTables = Arrays.stream(files) + .filter(file -> file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) + .count(); + return (int) countSsTables; } public Iterator> range( From 17676002e44310ca3ab5ddb3e8d31dd3a8a8dd41 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Fri, 27 Oct 2023 23:47:20 +0300 Subject: [PATCH 16/23] several fixes for codeclimate --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 3 +- .../vk/itmo/tyapuevdmitrij/MergeIterator.java | 51 +++++++++++-------- .../ru/vk/itmo/tyapuevdmitrij/Storage.java | 16 +++--- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 6586d3baf..6693963ae 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -4,6 +4,7 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import java.io.FileNotFoundException; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -103,7 +104,7 @@ public void compact() throws IOException { try { storage.deleteOldSsTables(ssTablePath); } catch (NoSuchFieldException e) { - throw new RuntimeException(e); + throw new FileNotFoundException(); } compacted = true; } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java index eb1e295e0..a82adcc43 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MergeIterator.java @@ -1,24 +1,27 @@ package ru.vk.itmo.tyapuevdmitrij; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; import java.util.Collection; import java.util.Comparator; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.PriorityQueue; -public class MergeIterator implements Iterator { +public class MergeIterator implements Iterator> { - private final PriorityQueue> priorityQueue; - private final Comparator comparator; - PeekIterator tableIterator; + private final PriorityQueue priorityQueue; + private final Comparator> comparator; + PeekIterator tableIterator; - private static class PeekIterator implements Iterator { + private static class PeekIterator implements Iterator> { public final int id; - private final Iterator delegate; - private T memorySegmentsEntry; + private final Iterator> delegate; + private Entry memorySegmentsEntry; - private PeekIterator(int id, Iterator delegate) { + private PeekIterator(int id, Iterator> delegate) { this.id = id; this.delegate = delegate; } @@ -32,16 +35,16 @@ public boolean hasNext() { } @Override - public T next() { + public Entry next() { if (!hasNext()) { throw new NoSuchElementException(); } - T peek = peek(); + Entry peek = peek(); this.memorySegmentsEntry = null; return peek; } - private T peek() { + private Entry peek() { if (memorySegmentsEntry == null) { if (!delegate.hasNext()) { return null; @@ -52,23 +55,24 @@ private T peek() { } } - public MergeIterator(Collection> iterators, Comparator comparator) { + public MergeIterator(Collection>> iterators, + Comparator> comparator) { this.comparator = comparator; - Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + Comparator peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); priorityQueue = new PriorityQueue<>( iterators.size(), peekComp.thenComparing(o -> -o.id) ); int id = 0; - for (Iterator iterator : iterators) { + for (Iterator> iterator : iterators) { if (iterator.hasNext()) { - priorityQueue.add(new PeekIterator<>(id++, iterator)); + priorityQueue.add(new PeekIterator(id++, iterator)); } } } - private PeekIterator peek() { + private PeekIterator peek() { while (tableIterator == null) { tableIterator = priorityQueue.poll(); if (tableIterator == null) { @@ -94,13 +98,13 @@ private PeekIterator peek() { private void peekFromPriorityQueue() { while (true) { - PeekIterator next = priorityQueue.peek(); + PeekIterator next = priorityQueue.peek(); if (next == null) { break; } int compare = comparator.compare(tableIterator.peek(), next.peek()); if (compare == 0) { - PeekIterator poll = priorityQueue.poll(); + PeekIterator poll = priorityQueue.poll(); if (poll != null) { poll.next(); if (poll.hasNext()) { @@ -113,7 +117,10 @@ private void peekFromPriorityQueue() { } } - protected boolean skip(T t) { + protected boolean skip(Entry entry) { + if (entry != null) { + return entry.value() == null; + } return false; } @@ -123,12 +130,12 @@ public boolean hasNext() { } @Override - public T next() { - PeekIterator entryIterator = peek(); + public Entry next() { + PeekIterator entryIterator = peek(); if (entryIterator == null) { throw new NoSuchElementException(); } - T next = entryIterator.next(); + Entry next = entryIterator.next(); this.tableIterator = null; if (entryIterator.hasNext()) { priorityQueue.add(entryIterator); diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java index ac13ff922..e9ddb7d31 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java @@ -57,17 +57,20 @@ public void deleteOldSsTables(Path ssTablePath) throws NoSuchFieldException { if (files == null) { throw new NoSuchFieldException(); } - boolean deleted = false; for (File file : files) { if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { - deleted = file.delete(); + try { + Files.delete(file.toPath()); + } catch (IOException e) { + throw new RuntimeException(e); + } } } boolean renamed; File remainingFile = files[0]; String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; renamed = remainingFile.renameTo(new File(newFilePath)); - if (!deleted && !renamed) { + if (!renamed) { throw new SecurityException(); } } @@ -257,12 +260,7 @@ public Iterator> range( } iterators.add(firstIterator); - return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, memorySegmentComparator)) { - @Override - protected boolean skip(Entry memorySegmentEntry) { - return memorySegmentEntry.value() == null; - } - }; + return new MergeIterator(iterators, Comparator.comparing(Entry::key, memorySegmentComparator)); } private Iterator> iterator(MemorySegment ssTable, MemorySegment from, MemorySegment to, From 028150ce8998194d3a6c0f8c60ffc50608b2eba2 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Sat, 28 Oct 2023 03:41:31 +0300 Subject: [PATCH 17/23] several fixes for codeclimate --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 12 +- .../ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java | 47 ++++++++ .../ru/vk/itmo/tyapuevdmitrij/Storage.java | 105 ++---------------- .../vk/itmo/tyapuevdmitrij/StorageHelper.java | 66 +++++++++++ 4 files changed, 126 insertions(+), 104 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java create mode 100644 src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 6693963ae..3bbef960b 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -4,7 +4,6 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.io.FileNotFoundException; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -34,7 +33,6 @@ public class InMemoryDao implements Dao> { new ConcurrentSkipListMap<>(memorySegmentComparator); private final Arena readArena; - private final Path ssTablePath; private long ssTablesEntryQuantity; private boolean compacted; @@ -93,7 +91,9 @@ public void compact() throws IOException { return; } Iterator> dataIterator = get(null, null); - MemorySegment buffer = storage.getWriteBufferToSsTable(getCompactionTableByteSize(), ssTablePath); + MemorySegment buffer = NmapBuffer.getWriteBufferToSsTable(getCompactionTableByteSize(), + ssTablePath, + storage.ssTablesQuantity); long bufferByteSize = buffer.byteSize(); buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, ssTablesEntryQuantity); long[] offsets = new long[2]; @@ -101,11 +101,7 @@ public void compact() throws IOException { while (dataIterator.hasNext()) { storage.writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); } - try { - storage.deleteOldSsTables(ssTablePath); - } catch (NoSuchFieldException e) { - throw new FileNotFoundException(); - } + StorageHelper.deleteOldSsTables(ssTablePath, storage.ssTablesQuantity); compacted = true; } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java new file mode 100644 index 000000000..e5fbcb211 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java @@ -0,0 +1,47 @@ +package ru.vk.itmo.tyapuevdmitrij; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.EnumSet; + +public class NmapBuffer { + + protected static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) { + MemorySegment buffer; + boolean created = false; + try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { + buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(ssTablePath), readArena); + created = true; + } catch (IOException e) { + buffer = null; + } finally { + if (!created) { + readArena.close(); + } + } + return buffer; + } + + protected static MemorySegment getWriteBufferToSsTable(Long writeBytes, + Path ssTablePath, + int ssTablesQuantity) throws IOException { + MemorySegment buffer; + Path path = ssTablePath.resolve(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity); + Arena writeArena = Arena.ofConfined(); + try (FileChannel channel = FileChannel.open(path, EnumSet.of(StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING))) { + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, + writeBytes, writeArena); + } + return buffer; + } +} + + diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java index e9ddb7d31..8ca30704f 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java @@ -2,81 +2,36 @@ import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; - -import java.io.File; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; -import java.util.EnumSet; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; public class Storage { protected List ssTables; - private long memTableEntriesSize; - private static final String SS_TABLE_FILE_NAME = "ssTable"; - protected int ssTablesQuantity; + protected static long memTableEntriesSize; + protected static final String SS_TABLE_FILE_NAME = "ssTable"; + protected final int ssTablesQuantity; public Storage(Path ssTablePath, Arena readArena) { - ssTablesQuantity = findSsTablesQuantity(ssTablePath); + ssTablesQuantity = StorageHelper.findSsTablesQuantity(ssTablePath); ssTables = new ArrayList<>(ssTablesQuantity); if (ssTablesQuantity != 0) { for (int i = 0; i < ssTablesQuantity; i++) { Path path = ssTablePath.resolve(SS_TABLE_FILE_NAME + i); - ssTables.add(getReadBufferFromSsTable(path, readArena)); - } - } - } - - private long getSsTableDataByteSize(Iterable> memTableEntries) { - long ssTableDataByteSize = 0; - long entriesCount = 0; - for (Entry entry : memTableEntries) { - ssTableDataByteSize += entry.key().byteSize(); - if (entry.value() != null) { - ssTableDataByteSize += entry.value().byteSize(); + ssTables.add(NmapBuffer.getReadBufferFromSsTable(path, readArena)); } - entriesCount++; - } - memTableEntriesSize = entriesCount; - return ssTableDataByteSize + entriesCount * Long.BYTES * 4L + Long.BYTES; - } - - public void deleteOldSsTables(Path ssTablePath) throws NoSuchFieldException { - File directory = new File(ssTablePath.toUri()); - File[] files = directory.listFiles(); - if (files == null) { - throw new NoSuchFieldException(); - } - for (File file : files) { - if (!file.getName().contains(SS_TABLE_FILE_NAME + ssTablesQuantity)) { - try { - Files.delete(file.toPath()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - boolean renamed; - File remainingFile = files[0]; - String newFilePath = remainingFile.getParent() + File.separator + SS_TABLE_FILE_NAME + 0; - renamed = remainingFile.renameTo(new File(newFilePath)); - if (!renamed) { - throw new SecurityException(); } } public void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, - Entry entry, long[] offsets) { + Entry entry, long... offsets) { buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); offsets[1] += Long.BYTES; buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.key().byteSize()); @@ -91,24 +46,10 @@ public void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, offsets[0] += entry.value().byteSize(); } - public MemorySegment getWriteBufferToSsTable(Long writeBytes, Path ssTablePath - ) throws IOException { - MemorySegment buffer; - Path path = ssTablePath.resolve(SS_TABLE_FILE_NAME + ssTablesQuantity); - Arena writeArena = Arena.ofConfined(); - try (FileChannel channel = FileChannel.open(path, EnumSet.of(StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING))) { - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, - writeBytes, writeArena); - } - return buffer; - } - public void save(Iterable> memTableEntries, Path ssTablePath) throws IOException { - MemorySegment buffer = getWriteBufferToSsTable(getSsTableDataByteSize(memTableEntries), - ssTablePath); + MemorySegment buffer = NmapBuffer.getWriteBufferToSsTable(StorageHelper.getSsTableDataByteSize(memTableEntries), + ssTablePath, + ssTablesQuantity); writeMemTableDataToFile(buffer, memTableEntries); } @@ -141,22 +82,6 @@ private void writeMemTableDataToFile(MemorySegment buffer, Iterable memorySegmentComparator) { long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); @@ -238,18 +163,6 @@ public Entry getSsTableDataByKey(MemorySegment key, return null; } - private int findSsTablesQuantity(Path ssTablePath) { - File dir = new File(ssTablePath.toUri()); - File[] files = dir.listFiles(); - if (files == null) { - return 0; - } - long countSsTables = Arrays.stream(files) - .filter(file -> file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) - .count(); - return (int) countSsTables; - } - public Iterator> range( Iterator> firstIterator, MemorySegment from, diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java new file mode 100644 index 000000000..320ebeaeb --- /dev/null +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -0,0 +1,66 @@ +package ru.vk.itmo.tyapuevdmitrij; + +import ru.vk.itmo.Entry; + +import java.io.File; +import java.lang.foreign.MemorySegment; +import java.nio.file.Path; +import java.util.Arrays; + +public class StorageHelper { + + protected static int findSsTablesQuantity(Path ssTablePath) { + File dir = new File(ssTablePath.toUri()); + File[] files = dir.listFiles(); + if (files == null) { + return 0; + } + long countSsTables = Arrays.stream(files) + .filter(file -> file.isFile() && file.getName().contains(Storage.SS_TABLE_FILE_NAME)) + .count(); + return (int) countSsTables; + } + + protected static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { + File directory = new File(ssTablePath.toUri()); + if (directory.exists() && directory.isDirectory()) { + if (directory.exists() && directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (!file.getName().contains(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity)) { + boolean deleted = file.delete(); + if (!deleted) { + throw new RuntimeException(); + } + } + } + } + } + File[] remainingFiles = directory.listFiles(); + if (remainingFiles != null && remainingFiles.length == 1) { + File remainingFile = remainingFiles[0]; + String newFilePath = remainingFile.getParent() + File.separator + Storage.SS_TABLE_FILE_NAME + 0; + boolean renamed = remainingFile.renameTo(new File(newFilePath)); + if (!renamed) { + throw new RuntimeException(); + } + } + + } + } + + protected static long getSsTableDataByteSize(Iterable> memTableEntries) { + long ssTableDataByteSize = 0; + long entriesCount = 0; + for (Entry entry : memTableEntries) { + ssTableDataByteSize += entry.key().byteSize(); + if (entry.value() != null) { + ssTableDataByteSize += entry.value().byteSize(); + } + entriesCount++; + } + Storage.memTableEntriesSize = entriesCount; + return ssTableDataByteSize + entriesCount * Long.BYTES * 4L + Long.BYTES; + } +} From 273a7874201fd25d883383ecf6c7051b997b8273 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Sat, 28 Oct 2023 04:04:06 +0300 Subject: [PATCH 18/23] several fixes for codeclimate --- .../vk/itmo/tyapuevdmitrij/InMemoryDao.java | 1 + .../ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java | 2 +- .../vk/itmo/tyapuevdmitrij/StorageHelper.java | 37 +++++++++++-------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java index 3bbef960b..c92e2972b 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java @@ -102,6 +102,7 @@ public void compact() throws IOException { storage.writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); } StorageHelper.deleteOldSsTables(ssTablePath, storage.ssTablesQuantity); + StorageHelper.renameCompactedSsTable(ssTablePath); compacted = true; } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java index e5fbcb211..13b8dded3 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java @@ -9,7 +9,7 @@ import java.nio.file.StandardOpenOption; import java.util.EnumSet; -public class NmapBuffer { +public abstract class NmapBuffer { protected static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) { MemorySegment buffer; diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java index 320ebeaeb..4fdc42490 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -3,11 +3,13 @@ import ru.vk.itmo.Entry; import java.io.File; +import java.io.IOException; import java.lang.foreign.MemorySegment; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -public class StorageHelper { +public abstract class StorageHelper { protected static int findSsTablesQuantity(Path ssTablePath) { File dir = new File(ssTablePath.toUri()); @@ -24,29 +26,34 @@ protected static int findSsTablesQuantity(Path ssTablePath) { protected static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { File directory = new File(ssTablePath.toUri()); if (directory.exists() && directory.isDirectory()) { - if (directory.exists() && directory.isDirectory()) { - File[] files = directory.listFiles(); - if (files != null) { - for (File file : files) { - if (!file.getName().contains(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity)) { - boolean deleted = file.delete(); - if (!deleted) { - throw new RuntimeException(); - } + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (!file.getName().contains(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity)) { + try { + Files.delete(file.toPath()); + } catch (IOException e) { + throw new SecurityException(e); } } } } + } + } + + protected static void renameCompactedSsTable(Path ssTablePath) { + File directory = new File(ssTablePath.toUri()); + boolean renamed = false; + if (directory.exists() && directory.isDirectory()) { File[] remainingFiles = directory.listFiles(); if (remainingFiles != null && remainingFiles.length == 1) { File remainingFile = remainingFiles[0]; String newFilePath = remainingFile.getParent() + File.separator + Storage.SS_TABLE_FILE_NAME + 0; - boolean renamed = remainingFile.renameTo(new File(newFilePath)); - if (!renamed) { - throw new RuntimeException(); - } + renamed = remainingFile.renameTo(new File(newFilePath)); } - + } + if (!renamed) { + throw new SecurityException(); } } From 6b80d949208e085e4c54278c816e156f57d8d6fe Mon Sep 17 00:00:00 2001 From: dmitrij Date: Sat, 28 Oct 2023 04:19:59 +0300 Subject: [PATCH 19/23] fix codeclimate --- .../ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java | 6 +++- .../vk/itmo/tyapuevdmitrij/StorageHelper.java | 32 +++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java index 13b8dded3..2ed156022 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java @@ -9,7 +9,11 @@ import java.nio.file.StandardOpenOption; import java.util.EnumSet; -public abstract class NmapBuffer { +public class NmapBuffer { + + private NmapBuffer() { + throw new IllegalStateException("Utility class"); + } protected static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) { MemorySegment buffer; diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java index 4fdc42490..fcf3cf9b3 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -9,7 +9,11 @@ import java.nio.file.Path; import java.util.Arrays; -public abstract class StorageHelper { +public class StorageHelper { + + private StorageHelper() { + throw new IllegalStateException("Utility class"); + } protected static int findSsTablesQuantity(Path ssTablePath) { File dir = new File(ssTablePath.toUri()); @@ -25,17 +29,19 @@ protected static int findSsTablesQuantity(Path ssTablePath) { protected static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { File directory = new File(ssTablePath.toUri()); - if (directory.exists() && directory.isDirectory()) { - File[] files = directory.listFiles(); - if (files != null) { - for (File file : files) { - if (!file.getName().contains(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity)) { - try { - Files.delete(file.toPath()); - } catch (IOException e) { - throw new SecurityException(e); - } - } + if (!directory.exists() && !directory.isDirectory()) { + return; + } + File[] files = directory.listFiles(); + if (files == null) { + return; + } + for (File file : files) { + if (!file.getName().contains(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity)) { + try { + Files.delete(file.toPath()); + } catch (IOException e) { + throw new SecurityException(e); } } } @@ -49,7 +55,7 @@ protected static void renameCompactedSsTable(Path ssTablePath) { if (remainingFiles != null && remainingFiles.length == 1) { File remainingFile = remainingFiles[0]; String newFilePath = remainingFile.getParent() + File.separator + Storage.SS_TABLE_FILE_NAME + 0; - renamed = remainingFile.renameTo(new File(newFilePath)); + renamed = remainingFile.renameTo(new File(newFilePath)); } } if (!renamed) { From 5603493e4142adf2a569fd8faee6468b568d044a Mon Sep 17 00:00:00 2001 From: dmitrij Date: Sat, 28 Oct 2023 04:24:55 +0300 Subject: [PATCH 20/23] fix codeclimate --- .../java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java | 10 +++++----- .../java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java index 2ed156022..c72fe62ec 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java @@ -9,13 +9,13 @@ import java.nio.file.StandardOpenOption; import java.util.EnumSet; -public class NmapBuffer { +public final class NmapBuffer { private NmapBuffer() { throw new IllegalStateException("Utility class"); } - protected static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) { + static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) { MemorySegment buffer; boolean created = false; try (FileChannel channel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { @@ -31,9 +31,9 @@ protected static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena return buffer; } - protected static MemorySegment getWriteBufferToSsTable(Long writeBytes, - Path ssTablePath, - int ssTablesQuantity) throws IOException { + static MemorySegment getWriteBufferToSsTable(Long writeBytes, + Path ssTablePath, + int ssTablesQuantity) throws IOException { MemorySegment buffer; Path path = ssTablePath.resolve(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity); Arena writeArena = Arena.ofConfined(); diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java index fcf3cf9b3..939de88a8 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -9,13 +9,13 @@ import java.nio.file.Path; import java.util.Arrays; -public class StorageHelper { +public final class StorageHelper { private StorageHelper() { throw new IllegalStateException("Utility class"); } - protected static int findSsTablesQuantity(Path ssTablePath) { + static int findSsTablesQuantity(Path ssTablePath) { File dir = new File(ssTablePath.toUri()); File[] files = dir.listFiles(); if (files == null) { @@ -27,7 +27,7 @@ protected static int findSsTablesQuantity(Path ssTablePath) { return (int) countSsTables; } - protected static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { + static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { File directory = new File(ssTablePath.toUri()); if (!directory.exists() && !directory.isDirectory()) { return; @@ -47,7 +47,7 @@ protected static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) } } - protected static void renameCompactedSsTable(Path ssTablePath) { + static void renameCompactedSsTable(Path ssTablePath) { File directory = new File(ssTablePath.toUri()); boolean renamed = false; if (directory.exists() && directory.isDirectory()) { @@ -63,7 +63,7 @@ protected static void renameCompactedSsTable(Path ssTablePath) { } } - protected static long getSsTableDataByteSize(Iterable> memTableEntries) { + static long getSsTableDataByteSize(Iterable> memTableEntries) { long ssTableDataByteSize = 0; long entriesCount = 0; for (Entry entry : memTableEntries) { From e6de16cd975f25f7f164bf2d811a29f2b287ffba Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 13 Nov 2023 05:45:56 +0300 Subject: [PATCH 21/23] Some fixes --- .../DaoFactoryImplementation.java | 6 +- ...InMemoryDao.java => MemorySegmentDao.java} | 61 +++++++++++---- .../ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java | 12 ++- .../ru/vk/itmo/tyapuevdmitrij/Storage.java | 74 +++++-------------- .../vk/itmo/tyapuevdmitrij/StorageHelper.java | 50 +++++++------ 5 files changed, 101 insertions(+), 102 deletions(-) rename src/main/java/ru/vk/itmo/tyapuevdmitrij/{InMemoryDao.java => MemorySegmentDao.java} (63%) diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java index 92ea67d93..7de4b3f90 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/DaoFactoryImplementation.java @@ -4,7 +4,7 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; import ru.vk.itmo.test.DaoFactory; -import ru.vk.itmo.tyapuevdmitrij.InMemoryDao; +import ru.vk.itmo.tyapuevdmitrij.MemorySegmentDao; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -15,12 +15,12 @@ public class DaoFactoryImplementation implements DaoFactory.Factory> createDao() { - return new InMemoryDao(); + return new MemorySegmentDao(); } @Override public Dao> createDao(Config config) { - return new InMemoryDao(config); + return new MemorySegmentDao(config); } @Override diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MemorySegmentDao.java similarity index 63% rename from src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java rename to src/main/java/ru/vk/itmo/tyapuevdmitrij/MemorySegmentDao.java index c92e2972b..12b7086b0 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/MemorySegmentDao.java @@ -9,14 +9,14 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.file.Path; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; -public class InMemoryDao implements Dao> { - - private static final Comparator memorySegmentComparator = (segment1, segment2) -> { +public class MemorySegmentDao implements Dao> { + private static final Comparator MEMORY_SEGMENT_COMPARATOR = (segment1, segment2) -> { long offset = segment1.mismatch(segment2); if (offset == -1) { return 0; @@ -30,7 +30,7 @@ public class InMemoryDao implements Dao> { return segment1.get(ValueLayout.JAVA_BYTE, offset) - segment2.get(ValueLayout.JAVA_BYTE, offset); }; private final ConcurrentNavigableMap> memTable = - new ConcurrentSkipListMap<>(memorySegmentComparator); + new ConcurrentSkipListMap<>(MEMORY_SEGMENT_COMPARATOR); private final Arena readArena; private final Path ssTablePath; @@ -38,13 +38,13 @@ public class InMemoryDao implements Dao> { private boolean compacted; private final Storage storage; - public InMemoryDao() { + public MemorySegmentDao() { ssTablePath = null; readArena = null; storage = null; } - public InMemoryDao(Config config) { + public MemorySegmentDao(Config config) { ssTablePath = config.basePath(); readArena = Arena.ofShared(); storage = new Storage(ssTablePath, readArena); @@ -52,7 +52,7 @@ public InMemoryDao(Config config) { @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return storage.range(getMemTableIterator(from, to), from, to, memorySegmentComparator); + return storage.range(getMemTableIterator(from, to), from, to, MEMORY_SEGMENT_COMPARATOR); } private Iterator> getMemTableIterator(MemorySegment from, MemorySegment to) { @@ -71,13 +71,25 @@ private Iterator> getMemTableIterator(MemorySegment from, M @Override public Entry get(MemorySegment key) { Entry value = memTable.get(key); - if (memTable.containsKey(key) && value.value() == null) { + if (value != null && value.value() == null) { return null; } if (value != null || storage.ssTables == null) { return value; } - return storage.getSsTableDataByKey(key, memorySegmentComparator); + Iterator> iterator = storage.range(Collections.emptyIterator(), + key, + null, + MEMORY_SEGMENT_COMPARATOR); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (MEMORY_SEGMENT_COMPARATOR.compare(next.key(), key) == 0) { + return next; + } + return null; } @Override @@ -91,17 +103,35 @@ public void compact() throws IOException { return; } Iterator> dataIterator = get(null, null); + Arena writeArena = Arena.ofConfined(); MemorySegment buffer = NmapBuffer.getWriteBufferToSsTable(getCompactionTableByteSize(), ssTablePath, - storage.ssTablesQuantity); + storage.ssTablesQuantity, + writeArena, + true); long bufferByteSize = buffer.byteSize(); buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, ssTablesEntryQuantity); - long[] offsets = new long[2]; - offsets[1] = bufferByteSize - Long.BYTES - ssTablesEntryQuantity * 2L * Long.BYTES; + long dataOffset = 0; + long indexOffset = bufferByteSize - Long.BYTES - ssTablesEntryQuantity * 2L * Long.BYTES; while (dataIterator.hasNext()) { - storage.writeEntryAndIndexesToCompactionTable(buffer, dataIterator.next(), offsets); + Entry entry = dataIterator.next(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + indexOffset += Long.BYTES; + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.key().byteSize()); + dataOffset += Long.BYTES; + MemorySegment.copy(entry.key(), 0, buffer, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + indexOffset += Long.BYTES; + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.value().byteSize()); + dataOffset += Long.BYTES; + MemorySegment.copy(entry.value(), 0, buffer, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); } - StorageHelper.deleteOldSsTables(ssTablePath, storage.ssTablesQuantity); + if (writeArena.scope().isAlive()) { + writeArena.close(); + } + StorageHelper.deleteOldSsTables(ssTablePath); StorageHelper.renameCompactedSsTable(ssTablePath); compacted = true; } @@ -109,9 +139,11 @@ public void compact() throws IOException { @Override public void close() throws IOException { if (compacted) { + readArena.close(); return; } if (memTable.isEmpty()) { + readArena.close(); return; } if (!readArena.scope().isAlive()) { @@ -135,5 +167,4 @@ private long getCompactionTableByteSize() { ssTablesEntryQuantity = countEntry; return compactionTableByteSize + countEntry * 4L * Long.BYTES + Long.BYTES; } - } diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java index c72fe62ec..e38322147 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/NmapBuffer.java @@ -33,10 +33,16 @@ static MemorySegment getReadBufferFromSsTable(Path ssTablePath, Arena readArena) static MemorySegment getWriteBufferToSsTable(Long writeBytes, Path ssTablePath, - int ssTablesQuantity) throws IOException { + int ssTablesQuantity, + Arena writeArena, + boolean compactionFlag) throws IOException { MemorySegment buffer; - Path path = ssTablePath.resolve(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity); - Arena writeArena = Arena.ofConfined(); + Path path; + if (compactionFlag) { + path = ssTablePath.resolve(StorageHelper.COMPACTED_FILE_NAME); + } else { + path = ssTablePath.resolve(StorageHelper.SS_TABLE_FILE_NAME + ssTablesQuantity); + } try (FileChannel channel = FileChannel.open(path, EnumSet.of(StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java index 8ca30704f..aa921cd8e 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/Storage.java @@ -2,6 +2,7 @@ import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; + import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -14,51 +15,39 @@ import java.util.NoSuchElementException; public class Storage { - protected List ssTables; - protected static long memTableEntriesSize; - protected static final String SS_TABLE_FILE_NAME = "ssTable"; + private final StorageHelper storageHelper; protected final int ssTablesQuantity; + protected List ssTables; public Storage(Path ssTablePath, Arena readArena) { + storageHelper = new StorageHelper(); ssTablesQuantity = StorageHelper.findSsTablesQuantity(ssTablePath); ssTables = new ArrayList<>(ssTablesQuantity); - if (ssTablesQuantity != 0) { - for (int i = 0; i < ssTablesQuantity; i++) { - Path path = ssTablePath.resolve(SS_TABLE_FILE_NAME + i); - ssTables.add(NmapBuffer.getReadBufferFromSsTable(path, readArena)); - } + for (int i = 0; i < ssTablesQuantity; i++) { + Path path = ssTablePath.resolve(StorageHelper.SS_TABLE_FILE_NAME + i); + ssTables.add(NmapBuffer.getReadBufferFromSsTable(path, readArena)); } } - public void writeEntryAndIndexesToCompactionTable(MemorySegment buffer, - Entry entry, long... offsets) { - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); - offsets[1] += Long.BYTES; - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.key().byteSize()); - offsets[0] += Long.BYTES; - MemorySegment.copy(entry.key(), 0, buffer, offsets[0], entry.key().byteSize()); - offsets[0] += entry.key().byteSize(); - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[1], offsets[0]); - offsets[1] += Long.BYTES; - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offsets[0], entry.value().byteSize()); - offsets[0] += Long.BYTES; - MemorySegment.copy(entry.value(), 0, buffer, offsets[0], entry.value().byteSize()); - offsets[0] += entry.value().byteSize(); - } - public void save(Iterable> memTableEntries, Path ssTablePath) throws IOException { - MemorySegment buffer = NmapBuffer.getWriteBufferToSsTable(StorageHelper.getSsTableDataByteSize(memTableEntries), + Arena writeArena = Arena.ofConfined(); + MemorySegment buffer = NmapBuffer.getWriteBufferToSsTable(storageHelper.getSsTableDataByteSize(memTableEntries), ssTablePath, - ssTablesQuantity); + ssTablesQuantity, + writeArena, + false); writeMemTableDataToFile(buffer, memTableEntries); + if (writeArena.scope().isAlive()) { + writeArena.close(); + } } private void writeMemTableDataToFile(MemorySegment buffer, Iterable> memTableEntries) { long offset = 0; long bufferByteSize = buffer.byteSize(); - long writeIndexPosition = bufferByteSize - memTableEntriesSize * 2L * Long.BYTES - Long.BYTES; + long writeIndexPosition = bufferByteSize - storageHelper.memTableEntriesCount * 2L * Long.BYTES - Long.BYTES; //write to the end of file size of memTable - buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, memTableEntriesSize); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, bufferByteSize - Long.BYTES, storageHelper.memTableEntriesCount); for (Entry entry : memTableEntries) { buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); // write keyByteSizeOffsetPosition to the end of buffer @@ -134,35 +123,6 @@ private MemorySegment getValueByOffset(MemorySegment ssTable, long offset) { return ssTable.asSlice(valueOffset, valueByteSize); } - public Entry getSsTableDataByKey(MemorySegment key, - Comparator memorySegmentComparator) { - for (int i = ssTables.size() - 1; i > -1; i--) { - MemorySegment ssTable = ssTables.get(i); - long memTableSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, ssTable.byteSize() - Long.BYTES); - long left = 0; - long right = memTableSize - 1L; - long lastKeyIndexOffset = ssTable.byteSize() - 3 * Long.BYTES; - while (left <= right) { - long mid = (right + left) >>> 1; - long midOffset = lastKeyIndexOffset - (memTableSize - 1L) * Long.BYTES * 2L + mid * 2L * Long.BYTES; - MemorySegment readKey = getKeyByOffset(ssTable, midOffset); - int res = memorySegmentComparator.compare(readKey, key); - if (res == 0) { - MemorySegment value = getValueByOffset(ssTable, midOffset + Long.BYTES); - if (value == null) { - return null; - } - return new BaseEntry<>(key, value); - } else if (res > 0) { - right = mid - 1; - } else { - left = mid + 1; - } - } - } - return null; - } - public Iterator> range( Iterator> firstIterator, MemorySegment from, diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java index 939de88a8..17f89da19 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -7,13 +7,13 @@ import java.lang.foreign.MemorySegment; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; +import java.nio.file.StandardCopyOption; -public final class StorageHelper { +public class StorageHelper { + protected static final String SS_TABLE_FILE_NAME = "ssTable"; - private StorageHelper() { - throw new IllegalStateException("Utility class"); - } + protected static final String COMPACTED_FILE_NAME = "compact"; + protected long memTableEntriesCount; static int findSsTablesQuantity(Path ssTablePath) { File dir = new File(ssTablePath.toUri()); @@ -21,13 +21,16 @@ static int findSsTablesQuantity(Path ssTablePath) { if (files == null) { return 0; } - long countSsTables = Arrays.stream(files) - .filter(file -> file.isFile() && file.getName().contains(Storage.SS_TABLE_FILE_NAME)) - .count(); + long countSsTables = 0L; + for (File file : files) { + if (file.isFile() && file.getName().contains(SS_TABLE_FILE_NAME)) { + countSsTables++; + } + } return (int) countSsTables; } - static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { + static void deleteOldSsTables(Path ssTablePath) { File directory = new File(ssTablePath.toUri()); if (!directory.exists() && !directory.isDirectory()) { return; @@ -37,7 +40,7 @@ static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { return; } for (File file : files) { - if (!file.getName().contains(Storage.SS_TABLE_FILE_NAME + ssTablesQuantity)) { + if (file.getName().contains(SS_TABLE_FILE_NAME)) { try { Files.delete(file.toPath()); } catch (IOException e) { @@ -48,22 +51,21 @@ static void deleteOldSsTables(Path ssTablePath, int ssTablesQuantity) { } static void renameCompactedSsTable(Path ssTablePath) { - File directory = new File(ssTablePath.toUri()); - boolean renamed = false; - if (directory.exists() && directory.isDirectory()) { - File[] remainingFiles = directory.listFiles(); - if (remainingFiles != null && remainingFiles.length == 1) { - File remainingFile = remainingFiles[0]; - String newFilePath = remainingFile.getParent() + File.separator + Storage.SS_TABLE_FILE_NAME + 0; - renamed = remainingFile.renameTo(new File(newFilePath)); - } - } - if (!renamed) { - throw new SecurityException(); + Path compactionFile = ssTablePath.resolve(COMPACTED_FILE_NAME); + Path newCompactionFile = ssTablePath.resolve(SS_TABLE_FILE_NAME + 0); + try { + Files.move( + compactionFile, + newCompactionFile, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } catch (IOException e) { + throw new SecurityException(e); } } - static long getSsTableDataByteSize(Iterable> memTableEntries) { + protected long getSsTableDataByteSize(Iterable> memTableEntries) { long ssTableDataByteSize = 0; long entriesCount = 0; for (Entry entry : memTableEntries) { @@ -73,7 +75,7 @@ static long getSsTableDataByteSize(Iterable> memTableEntrie } entriesCount++; } - Storage.memTableEntriesSize = entriesCount; + memTableEntriesCount = entriesCount; return ssTableDataByteSize + entriesCount * Long.BYTES * 4L + Long.BYTES; } } From 71ede3ad143a2b1b53f26c7a951929c91de01da5 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 13 Nov 2023 20:18:04 +0300 Subject: [PATCH 22/23] Last fixes --- .../itmo/tyapuevdmitrij/FilesException.java | 7 +++++++ .../vk/itmo/tyapuevdmitrij/StorageHelper.java | 21 +++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 src/main/java/ru/vk/itmo/tyapuevdmitrij/FilesException.java diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/FilesException.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/FilesException.java new file mode 100644 index 000000000..c84a4f84b --- /dev/null +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/FilesException.java @@ -0,0 +1,7 @@ +package ru.vk.itmo.tyapuevdmitrij; + +public class FilesException extends RuntimeException { + public FilesException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java index 17f89da19..c216fe2c1 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -16,8 +16,7 @@ public class StorageHelper { protected long memTableEntriesCount; static int findSsTablesQuantity(Path ssTablePath) { - File dir = new File(ssTablePath.toUri()); - File[] files = dir.listFiles(); + File[] files = getDirectoryFiles(ssTablePath); if (files == null) { return 0; } @@ -31,11 +30,7 @@ static int findSsTablesQuantity(Path ssTablePath) { } static void deleteOldSsTables(Path ssTablePath) { - File directory = new File(ssTablePath.toUri()); - if (!directory.exists() && !directory.isDirectory()) { - return; - } - File[] files = directory.listFiles(); + File[] files = getDirectoryFiles(ssTablePath); if (files == null) { return; } @@ -44,12 +39,20 @@ static void deleteOldSsTables(Path ssTablePath) { try { Files.delete(file.toPath()); } catch (IOException e) { - throw new SecurityException(e); + throw new FilesException("Can't delete file " + file.toPath(), e); } } } } + private static File[] getDirectoryFiles(Path ssTablePath) { + File directory = new File(ssTablePath.toUri()); + if (!directory.exists() || !directory.isDirectory()) { + return null; + } + return directory.listFiles(); + } + static void renameCompactedSsTable(Path ssTablePath) { Path compactionFile = ssTablePath.resolve(COMPACTED_FILE_NAME); Path newCompactionFile = ssTablePath.resolve(SS_TABLE_FILE_NAME + 0); @@ -61,7 +64,7 @@ static void renameCompactedSsTable(Path ssTablePath) { StandardCopyOption.REPLACE_EXISTING ); } catch (IOException e) { - throw new SecurityException(e); + throw new FilesException("Can't rename file", e); } } From 25a298576f68d5dfaeea6409b2b3dbc092c60749 Mon Sep 17 00:00:00 2001 From: dmitrij Date: Mon, 13 Nov 2023 20:31:29 +0300 Subject: [PATCH 23/23] Last fixes --- src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java index c216fe2c1..72ea38adf 100644 --- a/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/tyapuevdmitrij/StorageHelper.java @@ -17,7 +17,7 @@ public class StorageHelper { static int findSsTablesQuantity(Path ssTablePath) { File[] files = getDirectoryFiles(ssTablePath); - if (files == null) { + if (files.length == 0) { return 0; } long countSsTables = 0L; @@ -31,9 +31,6 @@ static int findSsTablesQuantity(Path ssTablePath) { static void deleteOldSsTables(Path ssTablePath) { File[] files = getDirectoryFiles(ssTablePath); - if (files == null) { - return; - } for (File file : files) { if (file.getName().contains(SS_TABLE_FILE_NAME)) { try { @@ -48,7 +45,7 @@ static void deleteOldSsTables(Path ssTablePath) { private static File[] getDirectoryFiles(Path ssTablePath) { File directory = new File(ssTablePath.toUri()); if (!directory.exists() || !directory.isDirectory()) { - return null; + return new File[0]; } return directory.listFiles(); }