From 3afcb42557b8b22886cc17caa05d242f049e9dbe Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Fri, 12 Jan 2024 10:31:07 -0800 Subject: [PATCH 01/28] [Tiered caching] Integrating ehcache disk cache Signed-off-by: Sagar Upadhyaya --- buildSrc/version.properties | 1 + .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 - .../licenses/slf4j-api-LICENSE.txt | 21 - .../licenses/slf4j-api-NOTICE.txt | 0 server/build.gradle | 3 + server/licenses/ehcache-3.10.8.jar.sha1 | 1 + server/licenses/ehcache-LICENSE.txt | 201 +++++++ server/licenses/ehcache-NOTICE.txt | 5 + .../licenses/slf4j-api-1.7.36.jar.sha1 | 0 .../licenses/slf4j-api-LICENSE.txt | 0 .../licenses/slf4j-api-NOTICE.txt | 0 .../org/opensearch/common/cache/ICache.java | 6 + .../common/cache/stats/CacheStats.java | 18 + .../common/cache/stats/package-info.java | 9 + .../common/cache/store/EhCacheDiskCache.java | 541 ++++++++++++++++++ .../cache/store/OpenSearchOnHeapCache.java | 23 +- .../cache/tier/TieredSpilloverCache.java | 35 +- .../org/opensearch/bootstrap/security.policy | 4 + .../cache/store/EhCacheDiskCacheTests.java | 469 +++++++++++++++ .../cache/tier/TieredSpilloverCacheTests.java | 11 + 35 files changed, 1321 insertions(+), 138 deletions(-) delete mode 100644 plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt delete mode 100644 plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt delete mode 100644 plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 delete mode 100644 plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt delete mode 100644 plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt delete mode 100644 plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 delete mode 100644 plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt delete mode 100644 plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt delete mode 100644 plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/slf4j-api-LICENSE.txt delete mode 100644 plugins/repository-azure/licenses/slf4j-api-NOTICE.txt delete mode 100644 plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt delete mode 100644 plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt delete mode 100644 plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/slf4j-api-LICENSE.txt delete mode 100644 plugins/repository-s3/licenses/slf4j-api-NOTICE.txt create mode 100644 server/licenses/ehcache-3.10.8.jar.sha1 create mode 100644 server/licenses/ehcache-LICENSE.txt create mode 100644 server/licenses/ehcache-NOTICE.txt rename {plugins/crypto-kms => server}/licenses/slf4j-api-1.7.36.jar.sha1 (100%) rename {plugins/crypto-kms => server}/licenses/slf4j-api-LICENSE.txt (100%) rename {plugins/crypto-kms => server}/licenses/slf4j-api-NOTICE.txt (100%) create mode 100644 server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java create mode 100644 server/src/main/java/org/opensearch/common/cache/stats/package-info.java create mode 100644 server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java create mode 100644 server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 3813750507f18..63b74d5f0200d 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -72,3 +72,4 @@ resteasy = 6.2.4.Final # opentelemetry dependencies opentelemetry = 1.32.0 opentelemetrysemconv = 1.23.1-alpha +ehcache = 3.10.8 diff --git a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt b/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 2be7689435062..0000000000000 --- a/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2022 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt b/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt b/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt b/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt b/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt b/plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt b/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-azure/licenses/slf4j-api-NOTICE.txt b/plugins/repository-azure/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt b/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt b/plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt b/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt deleted file mode 100644 index 8fda22f4d72f6..0000000000000 --- a/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2014 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/plugins/repository-s3/licenses/slf4j-api-NOTICE.txt b/plugins/repository-s3/licenses/slf4j-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/server/build.gradle b/server/build.gradle index e36498bf1038b..85f16cee83249 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -124,6 +124,9 @@ dependencies { api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" + api "org.ehcache:ehcache:${versions.ehcache}" + api "org.slf4j:slf4j-api:${versions.slf4j}" + testImplementation(project(":test:framework")) { // tests use the locally compiled version of server exclude group: 'org.opensearch', module: 'server' diff --git a/server/licenses/ehcache-3.10.8.jar.sha1 b/server/licenses/ehcache-3.10.8.jar.sha1 new file mode 100644 index 0000000000000..dee07e9238ebf --- /dev/null +++ b/server/licenses/ehcache-3.10.8.jar.sha1 @@ -0,0 +1 @@ +f0d50ede46609db78413ca7f4250d348a597b101 \ No newline at end of file diff --git a/server/licenses/ehcache-LICENSE.txt b/server/licenses/ehcache-LICENSE.txt new file mode 100644 index 0000000000000..8dada3edaf50d --- /dev/null +++ b/server/licenses/ehcache-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/ehcache-NOTICE.txt b/server/licenses/ehcache-NOTICE.txt new file mode 100644 index 0000000000000..1dbd38242cc98 --- /dev/null +++ b/server/licenses/ehcache-NOTICE.txt @@ -0,0 +1,5 @@ +Ehcache V3 +Copyright 2014-2023 Terracotta, Inc. + +The product includes software from the Apache Commons Lang project, +under the Apache License 2.0 (see: org.ehcache.impl.internal.classes.commonslang) diff --git a/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 b/server/licenses/slf4j-api-1.7.36.jar.sha1 similarity index 100% rename from plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 rename to server/licenses/slf4j-api-1.7.36.jar.sha1 diff --git a/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt b/server/licenses/slf4j-api-LICENSE.txt similarity index 100% rename from plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt rename to server/licenses/slf4j-api-LICENSE.txt diff --git a/plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt b/server/licenses/slf4j-api-NOTICE.txt similarity index 100% rename from plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt rename to server/licenses/slf4j-api-NOTICE.txt diff --git a/server/src/main/java/org/opensearch/common/cache/ICache.java b/server/src/main/java/org/opensearch/common/cache/ICache.java index c6ea5fca1a8fe..0eb778034e417 100644 --- a/server/src/main/java/org/opensearch/common/cache/ICache.java +++ b/server/src/main/java/org/opensearch/common/cache/ICache.java @@ -8,6 +8,8 @@ package org.opensearch.common.cache; +import org.opensearch.common.cache.stats.CacheStats; + /** * Represents a cache interface. * @param Type of key. @@ -31,4 +33,8 @@ public interface ICache { long count(); void refresh(); + + void close(); + + CacheStats stats(); } diff --git a/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java new file mode 100644 index 0000000000000..a952a2485ed23 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.stats; + +/** + * Interface for any cache specific stats. + * TODO: Add rest of stats like hits/misses. + */ +public interface CacheStats { + // Provides the number of entries in cache. + long count(); +} diff --git a/server/src/main/java/org/opensearch/common/cache/stats/package-info.java b/server/src/main/java/org/opensearch/common/cache/stats/package-info.java new file mode 100644 index 0000000000000..08aef5a9b3e88 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/stats/package-info.java @@ -0,0 +1,9 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/** Base package for stats related classes */ +package org.opensearch.common.cache.stats; diff --git a/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java b/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java new file mode 100644 index 0000000000000..f6315dc78eadf --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java @@ -0,0 +1,541 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.store; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.common.cache.LoadAwareCacheLoader; +import org.opensearch.common.cache.RemovalReason; +import org.opensearch.common.cache.stats.CacheStats; +import org.opensearch.common.cache.store.builders.StoreAwareCacheBuilder; +import org.opensearch.common.cache.store.enums.CacheStoreType; +import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; +import org.opensearch.common.collect.Tuple; +import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; + +import java.io.File; +import java.time.Duration; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.function.BiFunction; +import java.util.function.Supplier; + +import org.ehcache.Cache; +import org.ehcache.CachePersistenceException; +import org.ehcache.PersistentCacheManager; +import org.ehcache.config.builders.CacheConfigurationBuilder; +import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder; +import org.ehcache.config.builders.CacheManagerBuilder; +import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder; +import org.ehcache.config.builders.ResourcePoolsBuilder; +import org.ehcache.config.units.MemoryUnit; +import org.ehcache.event.CacheEvent; +import org.ehcache.event.CacheEventListener; +import org.ehcache.event.EventType; +import org.ehcache.expiry.ExpiryPolicy; +import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration; +import org.ehcache.spi.loaderwriter.CacheLoadingException; +import org.ehcache.spi.loaderwriter.CacheWritingException; + +/** + * This variant of disk cache uses Ehcache underneath. + * @param Type of key. + * @param Type of value. + * + * @opensearch.experimental + * + */ +public class EhCacheDiskCache implements StoreAwareCache { + + private static final Logger logger = LogManager.getLogger(EhCacheDiskCache.class); + + // A Cache manager can create many caches. + private final PersistentCacheManager cacheManager; + + // Disk cache + private Cache cache; + private final long maxWeightInBytes; + private final String storagePath; + + private final Class keyType; + + private final Class valueType; + + private final TimeValue expireAfterAccess; + + private final DiskCacheStats stats = new DiskCacheStats(); + + private final EhCacheEventListener ehCacheEventListener; + + private final String threadPoolAlias; + + private final Settings settings; + + private final static String DISK_CACHE_ALIAS = "ehDiskCache"; + + private final static String THREAD_POOL_ALIAS_PREFIX = "ehcachePool"; + + private final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB + + // Ehcache disk write minimum threads for its pool + public final Setting DISK_WRITE_MINIMUM_THREADS; + + // Ehcache disk write maximum threads for its pool + public final Setting DISK_WRITE_MAXIMUM_THREADS; + + // Not be to confused with number of disk segments, this is different. Defines + // distinct write queues created for disk store where a group of segments share a write queue. This is + // implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write + // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool. + public final Setting DISK_WRITE_CONCURRENCY; + + // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but + // will hold that many file pointers. Default is 16. + public final Setting DISK_SEGMENTS; + + private final StoreAwareCacheEventListener eventListener; + + /** + * Used in computeIfAbsent to synchronize loading of a given key. This is needed as ehcache doesn't provide a + * computeIfAbsent method. + */ + Map>> completableFutureMap = new ConcurrentHashMap<>(); + + private EhCacheDiskCache(Builder builder) { + this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null"); + this.valueType = Objects.requireNonNull(builder.valueType, "Value type shouldn't be null"); + this.expireAfterAccess = Objects.requireNonNull(builder.getExpireAfterAcess(), "ExpireAfterAccess value shouldn't " + "be null"); + this.maxWeightInBytes = builder.getMaxWeightInBytes(); + if (this.maxWeightInBytes <= MINIMUM_MAX_SIZE_IN_BYTES) { + throw new IllegalArgumentException("Ehcache Disk tier cache size should be greater than " + MINIMUM_MAX_SIZE_IN_BYTES); + } + this.storagePath = Objects.requireNonNull(builder.storagePath, "Storage path shouldn't be null"); + if (builder.threadPoolAlias == null || builder.threadPoolAlias.isBlank()) { + this.threadPoolAlias = THREAD_POOL_ALIAS_PREFIX + "DiskWrite"; + } else { + this.threadPoolAlias = builder.threadPoolAlias; + } + this.settings = Objects.requireNonNull(builder.settings, "Settings objects shouldn't be null"); + Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null"); + this.DISK_WRITE_MINIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tier.disk.ehcache.min_threads", 2, 1, 5); + this.DISK_WRITE_MAXIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tier.disk.ehcache.max_threads", 2, 1, 20); + // Default value is 1 within EhCache. + this.DISK_WRITE_CONCURRENCY = Setting.intSetting(builder.settingPrefix + ".tier.disk.ehcache.concurrency", 2, 1, 3); + // Default value is 16 within Ehcache. + this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + "tier.disk.ehcache.segments", 16, 1, 32); + this.cacheManager = buildCacheManager(); + Objects.requireNonNull(builder.getEventListener(), "Listener can't be null"); + this.eventListener = builder.getEventListener(); + this.ehCacheEventListener = new EhCacheEventListener(builder.getEventListener()); + this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder); + } + + private PersistentCacheManager buildCacheManager() { + // In case we use multiple ehCaches, we can define this cache manager at a global level. + return CacheManagerBuilder.newCacheManagerBuilder() + .with(CacheManagerBuilder.persistence(new File(storagePath))) + .using( + PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder() + .defaultPool(THREAD_POOL_ALIAS_PREFIX + "Default", 1, 3) // Default pool used for other tasks like + // event listeners + .pool(this.threadPoolAlias, DISK_WRITE_MINIMUM_THREADS.get(settings), DISK_WRITE_MAXIMUM_THREADS.get(settings)) + .build() + ) + .build(true); + } + + private Cache buildCache(Duration expireAfterAccess, Builder builder) { + return this.cacheManager.createCache( + DISK_CACHE_ALIAS, + CacheConfigurationBuilder.newCacheConfigurationBuilder( + this.keyType, + this.valueType, + ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B) + ).withExpiry(new ExpiryPolicy<>() { + @Override + public Duration getExpiryForCreation(K key, V value) { + return INFINITE; + } + + @Override + public Duration getExpiryForAccess(K key, Supplier value) { + return expireAfterAccess; + } + + @Override + public Duration getExpiryForUpdate(K key, Supplier oldValue, V newValue) { + return INFINITE; + } + }) + .withService(getListenerConfiguration(builder)) + .withService( + new OffHeapDiskStoreConfiguration( + this.threadPoolAlias, + DISK_WRITE_CONCURRENCY.get(settings), + DISK_SEGMENTS.get(settings) + ) + ) + ); + } + + private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder builder) { + CacheEventListenerConfigurationBuilder configurationBuilder = CacheEventListenerConfigurationBuilder.newEventListenerConfiguration( + this.ehCacheEventListener, + EventType.EVICTED, + EventType.EXPIRED, + EventType.REMOVED, + EventType.UPDATED, + EventType.CREATED + ).unordered(); + if (builder.isEventListenerModeSync) { + return configurationBuilder.synchronous(); + } else { + return configurationBuilder.asynchronous(); + } + } + + // Package private for testing + Map>> getCompletableFutureMap() { + return completableFutureMap; + } + + @Override + public V get(K key) { + if (key == null) { + throw new IllegalArgumentException("Key passed to ehcache disk cache was null."); + } + // Optimize it by adding key store. + V value; + try { + value = cache.get(key); + } catch (CacheLoadingException ex) { + throw new OpenSearchException("Exception occurred while trying to fetch item from ehcache disk cache"); + } + if (value != null) { + eventListener.onHit(key, value, CacheStoreType.DISK); + } else { + eventListener.onMiss(key, CacheStoreType.DISK); + } + return value; + } + + @Override + public void put(K key, V value) { + try { + cache.put(key, value); + } catch (CacheWritingException ex) { + throw new OpenSearchException("Exception occurred while put item to ehcache disk cache"); + } + } + + @Override + public V computeIfAbsent(K key, LoadAwareCacheLoader loader) throws Exception { + // Ehache doesn't provide any computeIfAbsent function. Exposes putIfAbsent but that works differently and is + // not performant in case there are multiple concurrent request for same key. Below is our own custom + // implementation of computeIfAbsent on top of ehcache. Inspired by OpenSearch Cache implementation. + V value = cache.get(key); + if (value == null) { + value = compute(key, loader); + } + if (!loader.isLoaded()) { + eventListener.onHit(key, value, CacheStoreType.DISK); + } else { + eventListener.onMiss(key, CacheStoreType.DISK); + } + return value; + } + + private V compute(K key, LoadAwareCacheLoader loader) throws Exception { + // A future that returns a pair of key/value. + CompletableFuture> completableFuture = new CompletableFuture<>(); + // Only one of the threads will succeed putting a future into map for the same key. + // Rest will fetch existing future. + CompletableFuture> future = completableFutureMap.putIfAbsent(key, completableFuture); + // Handler to handle results post processing. Takes a tuple or exception as an input and returns + // the value. Also before returning value, puts the value in cache. + BiFunction, Throwable, V> handler = (pair, ex) -> { + V value = null; + if (pair != null) { + cache.put(pair.v1(), pair.v2()); + value = pair.v2(); // Returning a value itself assuming that a next get should return the same. Should + // be safe to assume if we got no exception and reached here. + } + completableFutureMap.remove(key); // Remove key from map as not needed anymore. + return value; + }; + CompletableFuture completableValue; + if (future == null) { + future = completableFuture; + completableValue = future.handle(handler); + V value; + try { + value = loader.load(key); + } catch (Exception ex) { + future.completeExceptionally(ex); + throw new ExecutionException(ex); + } + if (value == null) { + NullPointerException npe = new NullPointerException("loader returned a null value"); + future.completeExceptionally(npe); + throw new ExecutionException(npe); + } else { + future.complete(new Tuple<>(key, value)); + } + + } else { + completableValue = future.handle(handler); + } + V value; + try { + value = completableValue.get(); + if (future.isCompletedExceptionally()) { + future.get(); // call get to force the exception to be thrown for other concurrent callers + throw new IllegalStateException("Future completed exceptionally but no error thrown"); + } + } catch (InterruptedException ex) { + throw new IllegalStateException(ex); + } + return value; + } + + @Override + public void invalidate(K key) { + // There seems to be an thread leak issue while calling this and then closing cache. + try { + cache.remove(key); + } catch (CacheWritingException ex) { + // Handle + throw new RuntimeException(ex); + } + } + + @Override + public void invalidateAll() { + // TODO + } + + @Override + public Iterable keys() { + return () -> new EhCacheKeyIterator<>(cache.iterator()); + } + + @Override + public long count() { + return stats.count(); + } + + @Override + public void refresh() { + // TODO: ehcache doesn't provide a way to refresh a cache. + } + + @Override + public CacheStoreType getTierType() { + return CacheStoreType.DISK; + } + + @Override + public void close() { + cacheManager.removeCache(DISK_CACHE_ALIAS); + cacheManager.close(); + try { + cacheManager.destroyCache(DISK_CACHE_ALIAS); + } catch (CachePersistenceException e) { + throw new OpenSearchException("Exception occurred while destroying ehcache and associated data", e); + } + } + + @Override + public CacheStats stats() { + return stats; + } + + /** + * Stats related to disk cache. + */ + class DiskCacheStats implements CacheStats { + private CounterMetric count = new CounterMetric(); + + @Override + public long count() { + return count.count(); + } + } + + /** + * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers. + * @param Type of key + * @param Type of value + */ + class EhCacheEventListener implements CacheEventListener { + + private final StoreAwareCacheEventListener eventListener; + + EhCacheEventListener(StoreAwareCacheEventListener eventListener) { + this.eventListener = eventListener; + } + + @Override + public void onEvent(CacheEvent event) { + switch (event.getType()) { + case CREATED: + stats.count.inc(); + this.eventListener.onCached(event.getKey(), event.getNewValue(), CacheStoreType.DISK); + assert event.getOldValue() == null; + break; + case EVICTED: + this.eventListener.onRemoval( + new StoreAwareCacheRemovalNotification<>( + event.getKey(), + event.getOldValue(), + RemovalReason.EVICTED, + CacheStoreType.DISK + ) + ); + stats.count.dec(); + assert event.getNewValue() == null; + break; + case REMOVED: + stats.count.dec(); + this.eventListener.onRemoval( + new StoreAwareCacheRemovalNotification<>( + event.getKey(), + event.getOldValue(), + RemovalReason.EXPLICIT, + CacheStoreType.DISK + ) + ); + assert event.getNewValue() == null; + break; + case EXPIRED: + this.eventListener.onRemoval( + new StoreAwareCacheRemovalNotification<>( + event.getKey(), + event.getOldValue(), + RemovalReason.INVALIDATED, + CacheStoreType.DISK + ) + ); + stats.count.dec(); + assert event.getNewValue() == null; + break; + case UPDATED: + break; + default: + break; + } + } + } + + /** + * This iterator wraps ehCache iterator and only iterates over its keys. + * @param Type of key + */ + class EhCacheKeyIterator implements Iterator { + + Iterator> iterator; + + EhCacheKeyIterator(Iterator> iterator) { + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public K next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return iterator.next().getKey(); + } + } + + /** + * Builder object to build Ehcache disk tier. + * @param Type of key + * @param Type of value + */ + public static class Builder extends StoreAwareCacheBuilder { + private Class keyType; + + private Class valueType; + + private String storagePath; + + private String threadPoolAlias; + + private Settings settings; + + private String diskCacheAlias; + + private String settingPrefix; + + // Provides capability to make ehCache event listener to run in sync mode. Used for testing too. + private boolean isEventListenerModeSync; + + public Builder() {} + + public EhCacheDiskCache.Builder setKeyType(Class keyType) { + this.keyType = keyType; + return this; + } + + public EhCacheDiskCache.Builder setValueType(Class valueType) { + this.valueType = valueType; + return this; + } + + public EhCacheDiskCache.Builder setStoragePath(String storagePath) { + this.storagePath = storagePath; + return this; + } + + public EhCacheDiskCache.Builder setThreadPoolAlias(String threadPoolAlias) { + this.threadPoolAlias = threadPoolAlias; + return this; + } + + public EhCacheDiskCache.Builder setSettings(Settings settings) { + this.settings = settings; + return this; + } + + public EhCacheDiskCache.Builder setDiskCacheAlias(String diskCacheAlias) { + this.diskCacheAlias = diskCacheAlias; + return this; + } + + public EhCacheDiskCache.Builder setSettingPrefix(String settingPrefix) { + // TODO: Do some basic validation. So that it doesn't end with "." etc. + this.settingPrefix = settingPrefix; + return this; + } + + public EhCacheDiskCache.Builder setIsEventListenerModeSync(boolean isEventListenerModeSync) { + this.isEventListenerModeSync = isEventListenerModeSync; + return this; + } + + public EhCacheDiskCache build() { + return new EhCacheDiskCache<>(this); + } + } +} diff --git a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java index c497c8dbb7ea9..5b9ff5921a01c 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java +++ b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java @@ -13,6 +13,7 @@ import org.opensearch.common.cache.LoadAwareCacheLoader; import org.opensearch.common.cache.RemovalListener; import org.opensearch.common.cache.RemovalNotification; +import org.opensearch.common.cache.stats.CacheStats; import org.opensearch.common.cache.store.builders.StoreAwareCacheBuilder; import org.opensearch.common.cache.store.enums.CacheStoreType; import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; @@ -30,6 +31,8 @@ public class OpenSearchOnHeapCache implements StoreAwareCache, Remov private final StoreAwareCacheEventListener eventListener; + private final CacheStats stats = new OpenSearchOnHeapCacheStats(); + public OpenSearchOnHeapCache(Builder builder) { CacheBuilder cacheBuilder = CacheBuilder.builder() .setMaximumWeight(builder.getMaxWeightInBytes()) @@ -88,7 +91,7 @@ public Iterable keys() { @Override public long count() { - return cache.count(); + return stats.count(); } @Override @@ -96,6 +99,14 @@ public void refresh() { cache.refresh(); } + @Override + public void close() {} + + @Override + public CacheStats stats() { + return stats; + } + @Override public CacheStoreType getTierType() { return CacheStoreType.ON_HEAP; @@ -113,6 +124,16 @@ public void onRemoval(RemovalNotification notification) { ); } + /** + * Stats for opensearch on heap cache. + */ + class OpenSearchOnHeapCacheStats implements CacheStats { + @Override + public long count() { + return cache.count(); + } + } + /** * Builder object * @param Type of key diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java index 8b432c9484aed..2216b90b69d69 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java @@ -11,6 +11,7 @@ import org.opensearch.common.cache.ICache; import org.opensearch.common.cache.LoadAwareCacheLoader; import org.opensearch.common.cache.RemovalReason; +import org.opensearch.common.cache.stats.CacheStats; import org.opensearch.common.cache.store.StoreAwareCache; import org.opensearch.common.cache.store.StoreAwareCacheRemovalNotification; import org.opensearch.common.cache.store.StoreAwareCacheValue; @@ -45,6 +46,7 @@ public class TieredSpilloverCache implements ICache, StoreAwareCache private final Optional> onDiskCache; private final StoreAwareCache onHeapCache; private final StoreAwareCacheEventListener listener; + private final CacheStats stats = new TieredSpillOverCacheStats(); ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); ReleasableLock readLock = new ReleasableLock(readWriteLock.readLock()); ReleasableLock writeLock = new ReleasableLock(readWriteLock.writeLock()); @@ -162,11 +164,7 @@ public Iterable keys() { @Override public long count() { - long totalCount = 0; - for (StoreAwareCache storeAwareCache : cacheList) { - totalCount += storeAwareCache.count(); - } - return totalCount; + return stats.count(); } @Override @@ -178,6 +176,18 @@ public void refresh() { } } + @Override + public void close() { + for (StoreAwareCache storeAwareCache : cacheList) { + storeAwareCache.close(); + } + } + + @Override + public CacheStats stats() { + return stats; + } + @Override public void onMiss(K key, CacheStoreType cacheStoreType) { // Misses for tiered cache are tracked here itself. @@ -234,6 +244,21 @@ private Function> getValueFromTieredCache(boolean tri }; } + /** + * Stats for tiered spillover cache. + */ + class TieredSpillOverCacheStats implements CacheStats { + + @Override + public long count() { + long totalCount = 0; + for (StoreAwareCache storeAwareCache : cacheList) { + totalCount += storeAwareCache.count(); + } + return totalCount; + } + } + /** * Builder object for tiered spillover cache. * @param Type of key diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 77cd0ab05278e..a07ea064928e3 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -188,4 +188,8 @@ grant { permission java.io.FilePermission "/sys/fs/cgroup/memory", "read"; permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read"; + // For ehcache + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; + }; diff --git a/server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java b/server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java new file mode 100644 index 0000000000000..12bffb93fc2f8 --- /dev/null +++ b/server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java @@ -0,0 +1,469 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.store; + +import org.opensearch.common.cache.LoadAwareCacheLoader; +import org.opensearch.common.cache.store.enums.CacheStoreType; +import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.test.OpenSearchSingleNodeTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Phaser; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.CoreMatchers.instanceOf; + +public class EhCacheDiskCacheTests extends OpenSearchSingleNodeTestCase { + + private static final int CACHE_SIZE_IN_BYTES = 1024 * 101; + private static final String SETTING_PREFIX = "indices.request.cache"; + + public void testBasicGetAndPut() throws IOException { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setSettingPrefix(SETTING_PREFIX) + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + int randomKeys = randomIntBetween(10, 100); + Map keyValueMap = new HashMap<>(); + for (int i = 0; i < randomKeys; i++) { + keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + for (Map.Entry entry : keyValueMap.entrySet()) { + ehcacheTest.put(entry.getKey(), entry.getValue()); + } + for (Map.Entry entry : keyValueMap.entrySet()) { + String value = ehcacheTest.get(entry.getKey()); + assertEquals(entry.getValue(), value); + } + assertEquals(randomKeys, mockEventListener.onCachedCount.get()); + assertEquals(randomKeys, mockEventListener.onHitCount.get()); + + // Validate misses + int expectedNumberOfMisses = randomIntBetween(10, 200); + for (int i = 0; i < expectedNumberOfMisses; i++) { + ehcacheTest.get(UUID.randomUUID().toString()); + } + + assertEquals(expectedNumberOfMisses, mockEventListener.onMissCount.get()); + ehcacheTest.close(); + } + } + + public void testConcurrentPut() throws Exception { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setSettingPrefix(SETTING_PREFIX) + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + int randomKeys = randomIntBetween(20, 100); + Thread[] threads = new Thread[randomKeys]; + Phaser phaser = new Phaser(randomKeys + 1); + CountDownLatch countDownLatch = new CountDownLatch(randomKeys); + Map keyValueMap = new HashMap<>(); + int j = 0; + for (int i = 0; i < randomKeys; i++) { + keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + for (Map.Entry entry : keyValueMap.entrySet()) { + threads[j] = new Thread(() -> { + phaser.arriveAndAwaitAdvance(); + ehcacheTest.put(entry.getKey(), entry.getValue()); + countDownLatch.countDown(); + }); + threads[j].start(); + j++; + } + phaser.arriveAndAwaitAdvance(); // Will trigger parallel puts above. + countDownLatch.await(); // Wait for all threads to finish + for (Map.Entry entry : keyValueMap.entrySet()) { + String value = ehcacheTest.get(entry.getKey()); + assertEquals(entry.getValue(), value); + } + assertEquals(randomKeys, mockEventListener.onCachedCount.get()); + ehcacheTest.close(); + } + } + + public void testEhcacheParallelGets() throws Exception { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setSettingPrefix(SETTING_PREFIX) + .setIsEventListenerModeSync(true) // For accurate count + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + int randomKeys = randomIntBetween(20, 100); + Thread[] threads = new Thread[randomKeys]; + Phaser phaser = new Phaser(randomKeys + 1); + CountDownLatch countDownLatch = new CountDownLatch(randomKeys); + Map keyValueMap = new HashMap<>(); + int j = 0; + for (int i = 0; i < randomKeys; i++) { + keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + for (Map.Entry entry : keyValueMap.entrySet()) { + ehcacheTest.put(entry.getKey(), entry.getValue()); + } + assertEquals(keyValueMap.size(), ehcacheTest.count()); + for (Map.Entry entry : keyValueMap.entrySet()) { + threads[j] = new Thread(() -> { + phaser.arriveAndAwaitAdvance(); + assertEquals(entry.getValue(), ehcacheTest.get(entry.getKey())); + countDownLatch.countDown(); + }); + threads[j].start(); + j++; + } + phaser.arriveAndAwaitAdvance(); // Will trigger parallel puts above. + countDownLatch.await(); // Wait for all threads to finish + assertEquals(randomKeys, mockEventListener.onHitCount.get()); + ehcacheTest.close(); + } + } + + public void testEhcacheKeyIterator() throws Exception { + Settings settings = Settings.builder().build(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setSettingPrefix(SETTING_PREFIX) + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(new MockEventListener<>()) + .build(); + + int randomKeys = randomIntBetween(2, 100); + Map keyValueMap = new HashMap<>(); + for (int i = 0; i < randomKeys; i++) { + keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + for (Map.Entry entry : keyValueMap.entrySet()) { + ehcacheTest.put(entry.getKey(), entry.getValue()); + } + Iterator keys = ehcacheTest.keys().iterator(); + int keysCount = 0; + while (keys.hasNext()) { + String key = keys.next(); + keysCount++; + assertNotNull(ehcacheTest.get(key)); + } + assertEquals(CacheStoreType.DISK, ehcacheTest.getTierType()); + assertEquals(keysCount, randomKeys); + ehcacheTest.close(); + } + } + + public void testEvictions() throws Exception { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setSettingPrefix(SETTING_PREFIX) + .setIsEventListenerModeSync(true) + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + + // Generate a string with 100 characters + String value = generateRandomString(100); + + // Trying to generate more than 100kb to cause evictions. + for (int i = 0; i < 1000; i++) { + String key = "Key" + i; + ehcacheTest.put(key, value); + } + assertTrue(mockEventListener.onRemovalCount.get() > 0); + ehcacheTest.close(); + } + } + + public void testComputeIfAbsentConcurrently() throws Exception { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setSettingPrefix(SETTING_PREFIX) + .setIsEventListenerModeSync(true) + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + + int numberOfRequest = 2;// randomIntBetween(200, 400); + String key = UUID.randomUUID().toString(); + String value = "dummy"; + Thread[] threads = new Thread[numberOfRequest]; + Phaser phaser = new Phaser(numberOfRequest + 1); + CountDownLatch countDownLatch = new CountDownLatch(numberOfRequest); + + List> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>(); + + // Try to hit different request with the same key concurrently. Verify value is only loaded once. + for (int i = 0; i < numberOfRequest; i++) { + threads[i] = new Thread(() -> { + LoadAwareCacheLoader loadAwareCacheLoader = new LoadAwareCacheLoader<>() { + boolean isLoaded; + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public String load(String key) { + isLoaded = true; + return value; + } + }; + loadAwareCacheLoaderList.add(loadAwareCacheLoader); + phaser.arriveAndAwaitAdvance(); + try { + assertEquals(value, ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader)); + } catch (Exception e) { + throw new RuntimeException(e); + } + countDownLatch.countDown(); + }); + threads[i].start(); + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); + int numberOfTimesValueLoaded = 0; + for (int i = 0; i < numberOfRequest; i++) { + if (loadAwareCacheLoaderList.get(i).isLoaded()) { + numberOfTimesValueLoaded++; + } + } + assertEquals(1, numberOfTimesValueLoaded); + assertEquals(0, ((EhCacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); + assertEquals(1, mockEventListener.onMissCount.get()); + assertEquals(1, mockEventListener.onCachedCount.get()); + assertEquals(numberOfRequest - 1, mockEventListener.onHitCount.get()); + ehcacheTest.close(); + } + } + + public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setSettingPrefix(SETTING_PREFIX) + .setIsEventListenerModeSync(true) + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + + int numberOfRequest = randomIntBetween(200, 400); + String key = UUID.randomUUID().toString(); + Thread[] threads = new Thread[numberOfRequest]; + Phaser phaser = new Phaser(numberOfRequest + 1); + CountDownLatch countDownLatch = new CountDownLatch(numberOfRequest); + + List> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>(); + + // Try to hit different request with the same key concurrently. Loader throws exception. + for (int i = 0; i < numberOfRequest; i++) { + threads[i] = new Thread(() -> { + LoadAwareCacheLoader loadAwareCacheLoader = new LoadAwareCacheLoader<>() { + boolean isLoaded; + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public String load(String key) throws Exception { + isLoaded = true; + throw new RuntimeException("Exception"); + } + }; + loadAwareCacheLoaderList.add(loadAwareCacheLoader); + phaser.arriveAndAwaitAdvance(); + assertThrows(ExecutionException.class, () -> ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader)); + countDownLatch.countDown(); + }); + threads[i].start(); + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); + + assertEquals(0, ((EhCacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); + ehcacheTest.close(); + } + } + + public void testComputeIfAbsentWithNullValueLoading() throws Exception { + Settings settings = Settings.builder().build(); + MockEventListener mockEventListener = new MockEventListener<>(); + try (NodeEnvironment env = newNodeEnvironment(settings)) { + StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) + .setThreadPoolAlias("ehcacheTest") + .setSettingPrefix(SETTING_PREFIX) + .setIsEventListenerModeSync(true) + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setExpireAfterAccess(TimeValue.MAX_VALUE) + .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) + .setEventListener(mockEventListener) + .build(); + + int numberOfRequest = randomIntBetween(200, 400); + String key = UUID.randomUUID().toString(); + Thread[] threads = new Thread[numberOfRequest]; + Phaser phaser = new Phaser(numberOfRequest + 1); + CountDownLatch countDownLatch = new CountDownLatch(numberOfRequest); + + List> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>(); + + // Try to hit different request with the same key concurrently. Loader throws exception. + for (int i = 0; i < numberOfRequest; i++) { + threads[i] = new Thread(() -> { + LoadAwareCacheLoader loadAwareCacheLoader = new LoadAwareCacheLoader<>() { + boolean isLoaded; + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public String load(String key) throws Exception { + isLoaded = true; + return null; + } + }; + loadAwareCacheLoaderList.add(loadAwareCacheLoader); + phaser.arriveAndAwaitAdvance(); + try { + ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader); + } catch (Exception ex) { + assertThat(ex.getCause(), instanceOf(NullPointerException.class)); + } + assertThrows(ExecutionException.class, () -> ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader)); + countDownLatch.countDown(); + }); + threads[i].start(); + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); + + assertEquals(0, ((EhCacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); + ehcacheTest.close(); + } + } + + private static String generateRandomString(int length) { + String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + StringBuilder randomString = new StringBuilder(length); + + for (int i = 0; i < length; i++) { + int index = (int) (Math.random() * characters.length()); + randomString.append(characters.charAt(index)); + } + + return randomString.toString(); + } + + // TODO: Remove this from here in final PR. + enum EventType { + ON_HIT, + ON_MISS, + ON_CACHED, + ON_REMOVAL; + } + + class MockEventListener implements StoreAwareCacheEventListener { + + AtomicInteger onMissCount = new AtomicInteger(); + AtomicInteger onHitCount = new AtomicInteger(); + AtomicInteger onCachedCount = new AtomicInteger(); + AtomicInteger onRemovalCount = new AtomicInteger(); + + MockEventListener() {} + + @Override + public void onMiss(K key, CacheStoreType cacheStoreType) { + assert cacheStoreType.equals(CacheStoreType.DISK); + onMissCount.incrementAndGet(); + } + + @Override + public void onRemoval(StoreAwareCacheRemovalNotification notification) { + assert notification.getCacheStoreType().equals(CacheStoreType.DISK); + onRemovalCount.incrementAndGet(); + } + + @Override + public void onHit(K key, V value, CacheStoreType cacheStoreType) { + assert cacheStoreType.equals(CacheStoreType.DISK); + onHitCount.incrementAndGet(); + } + + @Override + public void onCached(K key, V value, CacheStoreType cacheStoreType) { + assert cacheStoreType.equals(CacheStoreType.DISK); + onCachedCount.incrementAndGet(); + } + } +} diff --git a/server/src/test/java/org/opensearch/common/cache/tier/TieredSpilloverCacheTests.java b/server/src/test/java/org/opensearch/common/cache/tier/TieredSpilloverCacheTests.java index eb75244c6f8b1..cce0449dc88b8 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/TieredSpilloverCacheTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/TieredSpilloverCacheTests.java @@ -10,6 +10,7 @@ import org.opensearch.common.cache.LoadAwareCacheLoader; import org.opensearch.common.cache.RemovalReason; +import org.opensearch.common.cache.stats.CacheStats; import org.opensearch.common.cache.store.OpenSearchOnHeapCache; import org.opensearch.common.cache.store.StoreAwareCache; import org.opensearch.common.cache.store.StoreAwareCacheRemovalNotification; @@ -758,6 +759,16 @@ public long count() { @Override public void refresh() {} + @Override + public void close() { + + } + + @Override + public CacheStats stats() { + return null; + } + @Override public CacheStoreType getTierType() { return CacheStoreType.DISK; From 00f4545cc26b458ff4a94b83c678f3afe0df103c Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Tue, 16 Jan 2024 00:28:32 -0800 Subject: [PATCH 02/28] Adding suppressForbidden for File.io used by ehcache Signed-off-by: Sagar Upadhyaya --- .../org/opensearch/common/cache/store/EhCacheDiskCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java b/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java index f6315dc78eadf..80def5f260cc5 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java +++ b/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.common.SuppressForbidden; import org.opensearch.common.cache.LoadAwareCacheLoader; import org.opensearch.common.cache.RemovalReason; import org.opensearch.common.cache.stats.CacheStats; @@ -145,6 +146,7 @@ private EhCacheDiskCache(Builder builder) { this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder); } + @SuppressForbidden(reason = "Ehcache uses File.io") private PersistentCacheManager buildCacheManager() { // In case we use multiple ehCaches, we can define this cache manager at a global level. return CacheManagerBuilder.newCacheManagerBuilder() From 1a026a8db9439b6857e8f81c5a2c1b1d72ac8f40 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 22 Sep 2023 13:32:47 -0700 Subject: [PATCH 03/28] Implements a memory-efficient roaring bitmap-based keystore for use in disk cache tier Signed-off-by: Peter Alfonsi --- .gitignore | 3 +- server/build.gradle | 4 + server/licenses/RoaringBitmap-0.9.49.jar.sha1 | 1 + server/licenses/RoaringBitmap-LICENSE.txt | 191 ++++++++++++ server/licenses/RoaringBitmap-NOTICE.txt | 0 server/licenses/shims-0.9.49.jar.sha1 | 1 + server/licenses/shims-LICENSE.txt | 191 ++++++++++++ server/licenses/shims-NOTICE.txt | 0 .../opensearch/indices/KeyLookupStore.java | 133 ++++++++ .../indices/RBMIntKeyLookupStore.java | 264 ++++++++++++++++ .../opensearch/indices/RBMSizeEstimator.java | 131 ++++++++ .../indices/RBMIntKeyLookupStoreTests.java | 295 ++++++++++++++++++ 12 files changed, 1213 insertions(+), 1 deletion(-) create mode 100644 server/licenses/RoaringBitmap-0.9.49.jar.sha1 create mode 100644 server/licenses/RoaringBitmap-LICENSE.txt create mode 100644 server/licenses/RoaringBitmap-NOTICE.txt create mode 100644 server/licenses/shims-0.9.49.jar.sha1 create mode 100644 server/licenses/shims-LICENSE.txt create mode 100644 server/licenses/shims-NOTICE.txt create mode 100644 server/src/main/java/org/opensearch/indices/KeyLookupStore.java create mode 100644 server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java create mode 100644 server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java create mode 100644 server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java diff --git a/.gitignore b/.gitignore index 7514d55cc3c9a..291a63cdeef92 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,5 @@ testfixtures_shared/ .ci/jobs/ # build files generated -doc-tools/missing-doclet/bin/ \ No newline at end of file +doc-tools/missing-doclet/bin/ +server/src/main/java/org/opensearch/indices/KLSPerformanceTest.java diff --git a/server/build.gradle b/server/build.gradle index e36498bf1038b..5fe436ba84f66 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -124,6 +124,10 @@ dependencies { api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" + // roaring bitmaps + api 'org.roaringbitmap:RoaringBitmap:0.9.49' + runtimeOnly 'org.roaringbitmap:shims:0.9.49' // might fix complaining about ArraysShims? + testImplementation(project(":test:framework")) { // tests use the locally compiled version of server exclude group: 'org.opensearch', module: 'server' diff --git a/server/licenses/RoaringBitmap-0.9.49.jar.sha1 b/server/licenses/RoaringBitmap-0.9.49.jar.sha1 new file mode 100644 index 0000000000000..919a73c074b6a --- /dev/null +++ b/server/licenses/RoaringBitmap-0.9.49.jar.sha1 @@ -0,0 +1 @@ +b45b49c1ec5c5fc48580412d0ca635e1833110ea \ No newline at end of file diff --git a/server/licenses/RoaringBitmap-LICENSE.txt b/server/licenses/RoaringBitmap-LICENSE.txt new file mode 100644 index 0000000000000..a890d4a062fad --- /dev/null +++ b/server/licenses/RoaringBitmap-LICENSE.txt @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2013-2016 the RoaringBitmap authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/RoaringBitmap-NOTICE.txt b/server/licenses/RoaringBitmap-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/licenses/shims-0.9.49.jar.sha1 b/server/licenses/shims-0.9.49.jar.sha1 new file mode 100644 index 0000000000000..9e76614ca5207 --- /dev/null +++ b/server/licenses/shims-0.9.49.jar.sha1 @@ -0,0 +1 @@ +8bd7794fbdaa9536354dd2d8d961d9503beb9460 \ No newline at end of file diff --git a/server/licenses/shims-LICENSE.txt b/server/licenses/shims-LICENSE.txt new file mode 100644 index 0000000000000..a890d4a062fad --- /dev/null +++ b/server/licenses/shims-LICENSE.txt @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2013-2016 the RoaringBitmap authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/shims-NOTICE.txt b/server/licenses/shims-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java b/server/src/main/java/org/opensearch/indices/KeyLookupStore.java new file mode 100644 index 0000000000000..60e1386a460ec --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/KeyLookupStore.java @@ -0,0 +1,133 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +/** + * An interface for objects that hold an in-memory record of hashes of keys in the disk cache. + * These objects have some internal data structure which stores some transformation of added + * int values. The internal representations may have collisions. Example transformations include a modulo + * or -abs(value), or some combination. + */ +public interface KeyLookupStore { + + /** + * Transforms the input value into the internal representation for this keystore + * and adds it to the internal data structure. + * @param value The value to add. + * @return true if the value was added, false if it wasn't added because of a + * collision or if it was already present. + */ + boolean add(T value) throws Exception; + + /** + * Checks if the transformation of the value is in the keystore. + * @param value The value to check. + * @return true if the value was found, false otherwise. Due to collisions, false positives are + * possible, but there should be no false negatives unless forceRemove() is called. + */ + boolean contains(T value) throws Exception; + + /** + * Returns the transformed version of the input value, that would be used to stored it in the keystore. + * This transformation should be always be the same for a given instance. + * @param value The value to transform. + * @return The transformed value. + */ + T getInternalRepresentation(T value); + + /** + * Attempts to safely remove a value from the internal structure, maintaining the property that contains(value) + * will never return a false negative. If removing would lead to a false negative, the value won't be removed. + * Classes may not implement safe removal. + * @param value The value to attempt to remove. + * @return true if the value was removed, false if it wasn't. + */ + boolean remove(T value) throws Exception; + + /** + * Returns the number of distinct values stored in the internal data structure. + * Does not count values which weren't successfully added due to collisions. + * @return The number of values + */ + int getSize(); + + /** + * Returns the number of times add() has been run, including unsuccessful attempts. + * @return The number of adding attempts. + */ + int getTotalAdds(); + + /** + * Returns the number of times add() has returned false due to a collision. + * @return The number of collisions. + */ + int getCollisions(); + + + /** + * Checks if two values would collide after being transformed by this store's transformation. + * @param value1 The first value to compare. + * @param value2 The second value to compare. + * @return true if the transformations are equal, false otherwise. + */ + boolean isCollision(T value1, T value2); + + /** + * Returns an estimate of the store's memory usage. + * @return The memory usage, in MB + */ + long getMemorySizeInBytes(); + + /** + * Returns the cap for the store's memory usage. + * @return The cap, in bytes + */ + long getMemorySizeCapInBytes(); + + /** + * Returns whether the store is at memory capacity and can't accept more entries + */ + boolean isFull(); + + /** + * Deletes the internal data structure and regenerates it from the values passed in. + * Also resets all stats related to adding. + * @param newValues The keys that should be in the reset structure. + */ + void regenerateStore(T[] newValues) throws Exception; + + /** + * Deletes all keys and resets all stats related to adding. + */ + void clear() throws Exception; +} diff --git a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java new file mode 100644 index 0000000000000..3789989b5eaf1 --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java @@ -0,0 +1,264 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +import org.opensearch.common.metrics.CounterMetric; +import org.roaringbitmap.RoaringBitmap; + +import java.util.HashSet; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values. + * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28. + * It also maintains a hash set of values which have had collisions. Values which haven't had collisions can be + * safely removed from the store. The fraction of collided values should be low, + * about 0.5% for a store with 10^7 values and a modulo of 2^28. + * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap. + */ +public class RBMIntKeyLookupStore implements KeyLookupStore { + protected final int modulo; + protected class KeyStoreStats { + protected int size; + protected long memSizeCapInBytes; + protected CounterMetric numAddAttempts; + protected CounterMetric numCollisions; + protected boolean guaranteesNoFalseNegatives; + protected int maxNumEntries; + protected boolean atCapacity; + protected CounterMetric numRemovalAttempts; + protected CounterMetric numSuccessfulRemovals; + protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) { + this.size = 0; + this.numAddAttempts = new CounterMetric(); + this.numCollisions = new CounterMetric(); + this.memSizeCapInBytes = memSizeCapInBytes; + this.maxNumEntries = maxNumEntries; + this.atCapacity = false; + this.numRemovalAttempts = new CounterMetric(); + this.numSuccessfulRemovals = new CounterMetric(); + } + } + + protected KeyStoreStats stats; + protected RoaringBitmap rbm; + private HashSet collidedInts; + protected RBMSizeEstimator sizeEstimator; + protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + protected final Lock readLock = lock.readLock(); + protected final Lock writeLock = lock.writeLock(); + + RBMIntKeyLookupStore(int modulo, long memSizeCapInBytes) { + this.modulo = modulo; + sizeEstimator = new RBMSizeEstimator(modulo); + this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes)); + this.rbm = new RoaringBitmap(); + collidedInts = new HashSet<>(); + } + + protected int calculateMaxNumEntries(long memSizeCapInBytes) { + if (memSizeCapInBytes == 0) { + return Integer.MAX_VALUE; + } + return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes); + } + + protected final int transform(int value) { + return modulo == 0 ? value : value % modulo; + } + + protected void handleCollisions(int transformedValue) { + stats.numCollisions.inc(); + collidedInts.add(transformedValue); + } + + @Override + public boolean add(Integer value) throws Exception { + if (value == null) { + return false; + } + writeLock.lock(); + stats.numAddAttempts.inc(); + try { + if (stats.size == stats.maxNumEntries) { + stats.atCapacity = true; + return false; + } + int transformedValue = transform(value); + boolean alreadyContained = contains(value); + if (!alreadyContained) { + rbm.add(transformedValue); + stats.size++; + return true; + } + handleCollisions(transformedValue); + return false; + } finally { + writeLock.unlock(); + } + } + + @Override + public boolean contains(Integer value) throws Exception { + if (value == null) { + return false; + } + int transformedValue = transform(value); + readLock.lock(); + try { + return rbm.contains(transformedValue); + } finally { + readLock.unlock(); + } + } + + @Override + public Integer getInternalRepresentation(Integer value) { + if (value == null) { + return 0; + } + return Integer.valueOf(transform(value)); + } + + @Override + public boolean remove(Integer value) throws Exception { + if (value == null) { + return false; + } + int transformedValue = transform(value); + readLock.lock(); + try { + if (!contains(value)) { + return false; + } + stats.numRemovalAttempts.inc(); + if (collidedInts.contains(transformedValue)) { + return false; + } + } finally { + readLock.unlock(); + } + writeLock.lock(); + try { + rbm.remove(transformedValue); + stats.size--; + stats.numSuccessfulRemovals.inc(); + return true; + } finally { + writeLock.unlock(); + } + } + + @Override + public int getSize() { + readLock.lock(); + try { + return stats.size; + } finally { + readLock.unlock(); + } + } + + @Override + public int getTotalAdds() { + return (int) stats.numAddAttempts.count(); + } + + @Override + public int getCollisions() { + return (int) stats.numCollisions.count(); + } + + + @Override + public boolean isCollision(Integer value1, Integer value2) { + if (value1 == null || value2 == null) { + return false; + } + return transform(value1) == transform(value2); + } + + @Override + public long getMemorySizeInBytes() { + return sizeEstimator.getSizeInBytes(stats.size) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); + } + + @Override + public long getMemorySizeCapInBytes() { + return stats.memSizeCapInBytes; + } + + @Override + public boolean isFull() { + return stats.atCapacity; + } + + @Override + public void regenerateStore(Integer[] newValues) throws Exception { + rbm.clear(); + collidedInts = new HashSet<>(); + stats.size = 0; + stats.numAddAttempts = new CounterMetric(); + stats.numCollisions = new CounterMetric(); + stats.guaranteesNoFalseNegatives = true; + stats.numRemovalAttempts = new CounterMetric(); + stats.numSuccessfulRemovals = new CounterMetric(); + for (int i = 0; i < newValues.length; i++) { + if (newValues[i] != null) { + add(newValues[i]); + } + } + } + + + + @Override + public void clear() throws Exception { + regenerateStore(new Integer[]{}); + } + public int getNumRemovalAttempts() { + return (int) stats.numRemovalAttempts.count(); + } + + public int getNumSuccessfulRemovals() { + return (int) stats.numSuccessfulRemovals.count(); + } + + public boolean valueHasHadCollision(Integer value) { + if (value == null) { + return false; + } + return collidedInts.contains(transform(value)); + } +} diff --git a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java new file mode 100644 index 0000000000000..6e3b8e581ba9f --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +/** + * A class used to estimate roaring bitmap memory sizes (and hash set sizes). + * Values based on experiments with adding randomly distributed integers, which matches the use case for KeyLookupStore. + * In this use case, true values are much higher than an RBM's self-reported size, especially for small RBMs: see + * https://github.com/RoaringBitmap/RoaringBitmap/issues/257 + */ +public class RBMSizeEstimator { + public static final int BYTES_IN_MB = 1048576; + public static final double HASHSET_MEM_SLOPE = 6.46 * Math.pow(10, -5); + protected final double slope; + protected final double bufferMultiplier; + protected final double intercept; + + RBMSizeEstimator(int modulo) { + double[] memValues = calculateMemoryCoefficients(modulo); + this.bufferMultiplier = memValues[0]; + this.slope = memValues[1]; + this.intercept = memValues[2]; + } + + public static double[] calculateMemoryCoefficients(int modulo) { + // Sets up values to help estimate RBM size given a modulo + // Returns an array of {bufferMultiplier, slope, intercept} + + double modifiedModulo; + if (modulo == 0) { + modifiedModulo = 32.0; + } else { + modifiedModulo = Math.log(modulo) / Math.log(2); + } + // we "round up" the modulo to the nearest tested value + double highCutoff = 29.001; // Floating point makes 29 not work + double mediumCutoff = 28.0; + double lowCutoff = 26.0; + double bufferMultiplier = 1.0; + double slope; + double intercept; + if (modifiedModulo > highCutoff) { + // modulo > 2^29 + bufferMultiplier = 1.2; + slope = 0.637; + intercept = 3.091; + } else if (modifiedModulo > mediumCutoff) { + // 2^29 >= modulo > 2^28 + slope = 0.619; + intercept = 2.993; + } else if (modifiedModulo > lowCutoff) { + // 2^28 >= modulo > 2^26 + slope = 0.614; + intercept = 2.905; + } else { + slope = 0.628; + intercept = 2.603; + } + return new double[] { bufferMultiplier, slope, intercept }; + } + + public long getSizeInBytes(int numEntries) { + // Based on a linear fit in log-log space, so that we minimize the error as a proportion rather than as + // an absolute value. Should be within ~50% of the true value at worst, and should overestimate rather + // than underestimate the memory usage + return (long) ((long) Math.pow(numEntries, slope) * (long) Math.pow(10, intercept) * bufferMultiplier); + } + + public int getNumEntriesFromSizeInBytes(long sizeInBytes) { + // This function has some precision issues especially when composed with its inverse: + // numEntries = getNumEntriesFromSizeInBytes(getSizeInBytes(numEntries)) + // In this case the result can be off by up to a couple percent + // However, this shouldn't really matter as both functions are based on memory estimates with higher errors than a couple percent + // and this composition won't happen outside of tests + return (int) Math.pow(sizeInBytes / (bufferMultiplier * Math.pow(10, intercept)), 1 / slope); + + } + + public static long getSizeInBytesWithModulo(int numEntries, int modulo) { + double[] memValues = calculateMemoryCoefficients(modulo); + return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]); + } + + public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) { + double[] memValues = calculateMemoryCoefficients(modulo); + return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]); + } + + + protected static long convertMBToBytes(double valMB) { + return (long) (valMB * BYTES_IN_MB); + } + + protected static double convertBytesToMB(long valBytes) { + return (double) valBytes / BYTES_IN_MB; + } + + protected static long getHashsetMemSizeInBytes(int numEntries) { + return convertMBToBytes(HASHSET_MEM_SLOPE * numEntries); + } +} diff --git a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java new file mode 100644 index 0000000000000..c857c1ecab768 --- /dev/null +++ b/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java @@ -0,0 +1,295 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +import org.opensearch.common.Randomness; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase { + public void testInit() { + long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), memCap); + assertEquals(0, kls.getSize()); + assertEquals(memCap, kls.getMemorySizeCapInBytes()); + } + public void testTransformationLogic() throws Exception { + int modulo = (int) Math.pow(2, 29); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + int offset = 3; + for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design + int posValue = i * modulo + offset; + kls.add(posValue); + int negValue = -(i * modulo + offset); + kls.add(negValue); + } + assertEquals(2, kls.getSize()); + int[] testVals = new int[]{0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE}; + for (int value : testVals) { + assertTrue(kls.getInternalRepresentation(value) < modulo); + assertTrue(kls.getInternalRepresentation(value) > -modulo); + } + } + + public void testContains() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + for (int i = 0; i < 2000; i++) { + kls.add(i); + assertTrue(kls.contains(i)); + } + } + + public void testAddingStatsGetters() throws Exception { + int modulo = (int) Math.pow(2, 15); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, 0L); + kls.add(15); + kls.add(-15); + assertEquals(2, kls.getTotalAdds()); + assertEquals(0, kls.getCollisions()); + + int offset = 1; + for (int i = 0; i < 10; i++) { + kls.add(i * modulo + offset); + } + assertEquals(12, kls.getTotalAdds()); + assertEquals(9, kls.getCollisions()); + } + + public void testRegenerateStore() throws Exception { + int numToAdd = 10000000; + Random rand = Randomness.get(); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + for (int i = 0; i < numToAdd; i++) { + kls.add(i); + } + assertEquals(numToAdd, kls.getSize()); + Integer[] newVals = new Integer[1000]; // margin accounts for collisions + for (int j = 0; j < newVals.length; j++) { + newVals[j] = rand.nextInt(); + } + kls.regenerateStore(newVals); + assertTrue(Math.abs(kls.getSize() - newVals.length) < 3); // inexact due to collisions + + // test clear() + kls.clear(); + assertEquals(0, kls.getSize()); + } + + public void testAddingDuplicates() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + int numToAdd = 4820411; + for (int i = 0; i < numToAdd; i++) { + kls.add(i); + kls.add(i); + } + for (int j = 0; j < 1000; j++) { + kls.add(577); + } + assertEquals(numToAdd, kls.getSize()); + } + + public void testMemoryCapBlocksAdd() throws Exception { + int modulo = (int) Math.pow(2, 29); + for (int maxEntries: new int[]{2342000, 1000, 100000}) { + long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModulo(maxEntries, modulo); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes); + for (int j = 0; j < maxEntries + 1000; j++) { + kls.add(j); + } + assertTrue(Math.abs(maxEntries - kls.getSize()) < (double) maxEntries / 25); + // exact cap varies a small amount bc of floating point, especially when we use bytes instead of MB for calculations + // precision gets much worse when we compose the two functions, as we do here, but this wouldn't happen in an actual use case + } + } + + public void testConcurrency() throws Exception { + Random rand = Randomness.get(); + for (int j = 0; j < 5; j++) { // test with different numbers of threads + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + int numThreads = rand.nextInt(50) + 1; + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads); + // In this test we want to add the first 200K numbers and check they're all correctly there. + // We do some duplicates too to ensure those aren't incorrectly added. + int amountToAdd = 200000; + ArrayList> wasAdded = new ArrayList<>(amountToAdd); + ArrayList> duplicatesWasAdded = new ArrayList<>(); + for (int i = 0; i < amountToAdd; i++) { + wasAdded.add(null); + } + for (int i = 0; i < amountToAdd; i++) { + final int val = i; + Future fut = executor.submit(() -> { + boolean didAdd; + try { + didAdd = kls.add(val); + } catch (Exception e) { + throw new RuntimeException(e); + } + return didAdd; + }); + wasAdded.set(val, fut); + if (val % 1000 == 0) { + // do a duplicate add + Future duplicateFut = executor.submit(() -> { + boolean didAdd; + try { + didAdd = kls.add(val); + } catch (Exception e) { + throw new RuntimeException(e); + } + return didAdd; + }); + duplicatesWasAdded.add(duplicateFut); + } + } + int originalAdds = 0; + int duplicateAdds = 0; + for (Future fut : wasAdded) { + if (fut.get()) { + originalAdds++; + } + } + for (Future duplicateFut : duplicatesWasAdded) { + if (duplicateFut.get()) { + duplicateAdds++; + } + } + for (int i = 0; i < amountToAdd; i++) { + assertTrue(kls.contains(i)); + } + assertEquals(amountToAdd, originalAdds + duplicateAdds); + assertEquals(amountToAdd, kls.getSize()); + assertEquals(amountToAdd / 1000, kls.getCollisions()); + executor.shutdown(); + } + } + + public void testRemoveNoCollisions() throws Exception { + long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; + int numToAdd = 195000; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0, memCap); + // there should be no collisions for sequential positive numbers up to modulo + for (int i = 0; i < numToAdd; i++) { + kls.add(i); + } + for (int i = 0; i < 1000; i++) { + assertTrue(kls.remove(i)); + assertFalse(kls.contains(i)); + assertFalse(kls.valueHasHadCollision(i)); + } + assertEquals(numToAdd - 1000, kls.getSize()); + } + + public void testRemoveWithCollisions() throws Exception { + int modulo = (int) Math.pow(2, 26); + long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memCap); + for (int i = 0; i < 10; i++) { + kls.add(i); + if (i % 2 == 1) { + kls.add(-i); + assertFalse(kls.valueHasHadCollision(i)); + kls.add(i + modulo); + assertTrue(kls.valueHasHadCollision(i)); + } else { + assertFalse(kls.valueHasHadCollision(i)); + } + } + assertEquals(15, kls.getSize()); + for (int i = 0; i < 10; i++) { + boolean didRemove = kls.remove(i); + if (i % 2 == 1) { + // we expect a collision with i + modulo, so we can't remove + assertFalse(didRemove); + assertTrue(kls.contains(i)); + // but we should be able to remove -i + boolean didRemoveNegative = kls.remove(-i); + assertTrue(didRemoveNegative); + assertFalse(kls.contains(-i)); + } else { + // we expect no collision + assertTrue(didRemove); + assertFalse(kls.contains(i)); + assertFalse(kls.valueHasHadCollision(i)); + } + } + assertEquals(5, kls.getSize()); + int offset = 12; + kls.add(offset); + for (int j = 1; j < 5; j++) { + kls.add(offset + j * modulo); + } + assertEquals(6, kls.getSize()); + assertFalse(kls.remove(offset + modulo)); + assertTrue(kls.valueHasHadCollision(offset + 15 * modulo)); + assertTrue(kls.contains(offset + 17 * modulo)); + } + + public void testNullInputs() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + assertFalse(kls.add(null)); + assertFalse(kls.contains(null)); + assertEquals(0, (int) kls.getInternalRepresentation(null)); + assertFalse(kls.remove(null)); + assertFalse(kls.isCollision(null, null)); + assertEquals(0, kls.getTotalAdds()); + Integer[] newVals = new Integer[]{1, 17, -2, null, -4, null}; + kls.regenerateStore(newVals); + assertEquals(4, kls.getSize()); + } + + public void testMemoryCapValueInitialization() { + double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 }; + double[] expectedMultipliers = new double[] { 1.2, 1.2, 1.2, 1, 1, 1 }; + double[] expectedSlopes = new double[] { 0.637, 0.637, 0.637, 0.619, 0.614, 0.629 }; + double[] expectedIntercepts = new double[] { 3.091, 3.091, 3.091, 2.993, 2.905, 2.603 }; + long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB; + double delta = 0.01; + for (int i = 0; i < logModulos.length; i++) { + int modulo = 0; + if (logModulos[i] != 0) { + modulo = (int) Math.pow(2, logModulos[i]); + } + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes); + assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0); + assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta); + assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta); + assertEquals(expectedIntercepts[i], kls.sizeEstimator.intercept, delta); + } + + } +} From f25a2dc0797824c2d91ed2fd067d6c734e87e344 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Thu, 2 Nov 2023 16:30:20 -0700 Subject: [PATCH 04/28] Addressed Sagar's comments besides new counter/arraylist setup for removing keys Signed-off-by: Peter Alfonsi --- .../cache/tier/keystore}/KeyLookupStore.java | 5 +- .../cache/tier/keystore/KeyStoreStats.java | 40 ++++++++ .../tier/keystore}/RBMIntKeyLookupStore.java | 98 ++++++++++--------- .../tier/keystore}/RBMSizeEstimator.java | 10 +- .../keystore}/RBMIntKeyLookupStoreTests.java | 67 +++++++------ 5 files changed, 137 insertions(+), 83 deletions(-) rename server/src/main/java/org/opensearch/{indices => common/cache/tier/keystore}/KeyLookupStore.java (98%) create mode 100644 server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java rename server/src/main/java/org/opensearch/{indices => common/cache/tier/keystore}/RBMIntKeyLookupStore.java (77%) rename server/src/main/java/org/opensearch/{indices => common/cache/tier/keystore}/RBMSizeEstimator.java (92%) rename server/src/test/java/org/opensearch/{indices => common/cache/tier/keystore}/RBMIntKeyLookupStoreTests.java (80%) diff --git a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java similarity index 98% rename from server/src/main/java/org/opensearch/indices/KeyLookupStore.java rename to server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java index 60e1386a460ec..552698e0293eb 100644 --- a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.indices; +package org.opensearch.common.cache.tier.keystore; /** * An interface for objects that hold an in-memory record of hashes of keys in the disk cache. @@ -85,7 +85,7 @@ public interface KeyLookupStore { * Returns the number of times add() has been run, including unsuccessful attempts. * @return The number of adding attempts. */ - int getTotalAdds(); + int getAddAttempts(); /** * Returns the number of times add() has returned false due to a collision. @@ -93,7 +93,6 @@ public interface KeyLookupStore { */ int getCollisions(); - /** * Checks if two values would collide after being transformed by this store's transformation. * @param value1 The first value to compare. diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java new file mode 100644 index 0000000000000..b5c95b134990c --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.tier.keystore; + +import org.opensearch.common.metrics.CounterMetric; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A stats holder for use in KeyLookupStore implementations. + * Getters should be exposed by the KeyLookupStore which uses it. + */ +public class KeyStoreStats { + protected CounterMetric size; + protected long memSizeCapInBytes; + protected CounterMetric numAddAttempts; + protected CounterMetric numCollisions; + protected boolean guaranteesNoFalseNegatives; + protected final int maxNumEntries; + protected AtomicBoolean atCapacity; + protected CounterMetric numRemovalAttempts; + protected CounterMetric numSuccessfulRemovals; + + protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) { + this.size = new CounterMetric(); + this.numAddAttempts = new CounterMetric(); + this.numCollisions = new CounterMetric(); + this.memSizeCapInBytes = memSizeCapInBytes; + this.maxNumEntries = maxNumEntries; + this.atCapacity = new AtomicBoolean(false); + this.numRemovalAttempts = new CounterMetric(); + this.numSuccessfulRemovals = new CounterMetric(); + } +} diff --git a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java similarity index 77% rename from server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java rename to server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index 3789989b5eaf1..7e91a64758919 100644 --- a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -30,15 +30,16 @@ * GitHub history for details. */ -package org.opensearch.indices; +package org.opensearch.common.cache.tier.keystore; import org.opensearch.common.metrics.CounterMetric; -import org.roaringbitmap.RoaringBitmap; import java.util.HashSet; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.roaringbitmap.RoaringBitmap; + /** * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values. * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28. @@ -48,30 +49,29 @@ * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap. */ public class RBMIntKeyLookupStore implements KeyLookupStore { - protected final int modulo; - protected class KeyStoreStats { - protected int size; - protected long memSizeCapInBytes; - protected CounterMetric numAddAttempts; - protected CounterMetric numCollisions; - protected boolean guaranteesNoFalseNegatives; - protected int maxNumEntries; - protected boolean atCapacity; - protected CounterMetric numRemovalAttempts; - protected CounterMetric numSuccessfulRemovals; - protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) { - this.size = 0; - this.numAddAttempts = new CounterMetric(); - this.numCollisions = new CounterMetric(); - this.memSizeCapInBytes = memSizeCapInBytes; - this.maxNumEntries = maxNumEntries; - this.atCapacity = false; - this.numRemovalAttempts = new CounterMetric(); - this.numSuccessfulRemovals = new CounterMetric(); + /** + * An enum representing modulo values for use in the keystore + */ + public enum KeystoreModuloValue { + NONE(0), // No modulo applied + TWO_TO_THIRTY_ONE((int) Math.pow(2, 31)), + TWO_TO_TWENTY_NINE((int) Math.pow(2, 29)), // recommended value + TWO_TO_TWENTY_EIGHT((int) Math.pow(2, 28)), + TWO_TO_TWENTY_SIX((int) Math.pow(2, 26)); + + private final int value; + + private KeystoreModuloValue(int value) { + this.value = value; + } + + public int getValue() { + return this.value; } } - protected KeyStoreStats stats; + protected final int modulo; + KeyStoreStats stats; protected RoaringBitmap rbm; private HashSet collidedInts; protected RBMSizeEstimator sizeEstimator; @@ -79,8 +79,13 @@ protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) { protected final Lock readLock = lock.readLock(); protected final Lock writeLock = lock.writeLock(); - RBMIntKeyLookupStore(int modulo, long memSizeCapInBytes) { - this.modulo = modulo; + // Default constructor sets modulo = 2^28 + public RBMIntKeyLookupStore(long memSizeCapInBytes) { + this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes); + } + + public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) { + this.modulo = moduloValue.getValue(); sizeEstimator = new RBMSizeEstimator(modulo); this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes)); this.rbm = new RoaringBitmap(); @@ -94,11 +99,11 @@ protected int calculateMaxNumEntries(long memSizeCapInBytes) { return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes); } - protected final int transform(int value) { + private final int transform(int value) { return modulo == 0 ? value : value % modulo; } - protected void handleCollisions(int transformedValue) { + private void handleCollisions(int transformedValue) { stats.numCollisions.inc(); collidedInts.add(transformedValue); } @@ -108,18 +113,21 @@ public boolean add(Integer value) throws Exception { if (value == null) { return false; } - writeLock.lock(); stats.numAddAttempts.inc(); + if (stats.size.count() == stats.maxNumEntries) { + stats.atCapacity.set(true); + return false; + } + int transformedValue = transform(value); + + writeLock.lock(); try { - if (stats.size == stats.maxNumEntries) { - stats.atCapacity = true; - return false; - } - int transformedValue = transform(value); - boolean alreadyContained = contains(value); + boolean alreadyContained; + // saves calling transform() an additional time + alreadyContained = rbm.contains(transformedValue); if (!alreadyContained) { rbm.add(transformedValue); - stats.size++; + stats.size.inc(); return true; } handleCollisions(transformedValue); @@ -159,7 +167,7 @@ public boolean remove(Integer value) throws Exception { int transformedValue = transform(value); readLock.lock(); try { - if (!contains(value)) { + if (!rbm.contains(transformedValue)) { // saves additional transform() call return false; } stats.numRemovalAttempts.inc(); @@ -172,7 +180,7 @@ public boolean remove(Integer value) throws Exception { writeLock.lock(); try { rbm.remove(transformedValue); - stats.size--; + stats.size.dec(); stats.numSuccessfulRemovals.inc(); return true; } finally { @@ -184,14 +192,14 @@ public boolean remove(Integer value) throws Exception { public int getSize() { readLock.lock(); try { - return stats.size; + return (int) stats.size.count(); } finally { readLock.unlock(); } } @Override - public int getTotalAdds() { + public int getAddAttempts() { return (int) stats.numAddAttempts.count(); } @@ -200,7 +208,6 @@ public int getCollisions() { return (int) stats.numCollisions.count(); } - @Override public boolean isCollision(Integer value1, Integer value2) { if (value1 == null || value2 == null) { @@ -211,7 +218,7 @@ public boolean isCollision(Integer value1, Integer value2) { @Override public long getMemorySizeInBytes() { - return sizeEstimator.getSizeInBytes(stats.size) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); + return sizeEstimator.getSizeInBytes((int) stats.size.count()) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); } @Override @@ -221,14 +228,14 @@ public long getMemorySizeCapInBytes() { @Override public boolean isFull() { - return stats.atCapacity; + return stats.atCapacity.get(); } @Override public void regenerateStore(Integer[] newValues) throws Exception { rbm.clear(); collidedInts = new HashSet<>(); - stats.size = 0; + stats.size = new CounterMetric(); stats.numAddAttempts = new CounterMetric(); stats.numCollisions = new CounterMetric(); stats.guaranteesNoFalseNegatives = true; @@ -241,12 +248,11 @@ public void regenerateStore(Integer[] newValues) throws Exception { } } - - @Override public void clear() throws Exception { - regenerateStore(new Integer[]{}); + regenerateStore(new Integer[] {}); } + public int getNumRemovalAttempts() { return (int) stats.numRemovalAttempts.count(); } diff --git a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java similarity index 92% rename from server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java rename to server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java index 6e3b8e581ba9f..e07c645bad0cf 100644 --- a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java @@ -30,7 +30,9 @@ * GitHub history for details. */ -package org.opensearch.indices; +package org.opensearch.common.cache.tier.keystore; + +import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore; /** * A class used to estimate roaring bitmap memory sizes (and hash set sizes). @@ -111,12 +113,16 @@ public static long getSizeInBytesWithModulo(int numEntries, int modulo) { return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]); } + public static long getSizeInBytesWithModuloValue(int numEntries, RBMIntKeyLookupStore.KeystoreModuloValue moduloValue) { + double[] memValues = calculateMemoryCoefficients(moduloValue.getValue()); + return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]); + } + public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) { double[] memValues = calculateMemoryCoefficients(modulo); return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]); } - protected static long convertMBToBytes(double valMB) { return (long) (valMB * BYTES_IN_MB); } diff --git a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java similarity index 80% rename from server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java rename to server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index c857c1ecab768..08ab08462373b 100644 --- a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -29,9 +29,11 @@ * GitHub history for details. */ -package org.opensearch.indices; +package org.opensearch.common.cache.tier.keystore; import org.opensearch.common.Randomness; +import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore; +import org.opensearch.common.cache.tier.keystore.RBMSizeEstimator; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; @@ -43,13 +45,15 @@ public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase { public void testInit() { long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB; - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), memCap); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap); assertEquals(0, kls.getSize()); + assertEquals(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_EIGHT.getValue(), kls.modulo); assertEquals(memCap, kls.getMemorySizeCapInBytes()); } + public void testTransformationLogic() throws Exception { int modulo = (int) Math.pow(2, 29); - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); int offset = 3; for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design int posValue = i * modulo + offset; @@ -58,7 +62,7 @@ public void testTransformationLogic() throws Exception { kls.add(negValue); } assertEquals(2, kls.getSize()); - int[] testVals = new int[]{0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE}; + int[] testVals = new int[] { 0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE }; for (int value : testVals) { assertTrue(kls.getInternalRepresentation(value) < modulo); assertTrue(kls.getInternalRepresentation(value) > -modulo); @@ -66,7 +70,7 @@ public void testTransformationLogic() throws Exception { } public void testContains() throws Exception { - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); for (int i = 0; i < 2000; i++) { kls.add(i); assertTrue(kls.contains(i)); @@ -74,25 +78,25 @@ public void testContains() throws Exception { } public void testAddingStatsGetters() throws Exception { - int modulo = (int) Math.pow(2, 15); - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, 0L); + RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L); kls.add(15); kls.add(-15); - assertEquals(2, kls.getTotalAdds()); + assertEquals(2, kls.getAddAttempts()); assertEquals(0, kls.getCollisions()); int offset = 1; for (int i = 0; i < 10; i++) { - kls.add(i * modulo + offset); + kls.add(i * moduloValue.getValue() + offset); } - assertEquals(12, kls.getTotalAdds()); + assertEquals(12, kls.getAddAttempts()); assertEquals(9, kls.getCollisions()); } public void testRegenerateStore() throws Exception { int numToAdd = 10000000; Random rand = Randomness.get(); - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); for (int i = 0; i < numToAdd; i++) { kls.add(i); } @@ -110,7 +114,7 @@ public void testRegenerateStore() throws Exception { } public void testAddingDuplicates() throws Exception { - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0L); int numToAdd = 4820411; for (int i = 0; i < numToAdd; i++) { kls.add(i); @@ -123,10 +127,10 @@ public void testAddingDuplicates() throws Exception { } public void testMemoryCapBlocksAdd() throws Exception { - int modulo = (int) Math.pow(2, 29); - for (int maxEntries: new int[]{2342000, 1000, 100000}) { - long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModulo(maxEntries, modulo); - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes); + RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE; + for (int maxEntries : new int[] { 2342000, 1000, 100000 }) { + long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes); for (int j = 0; j < maxEntries + 1000; j++) { kls.add(j); } @@ -139,7 +143,7 @@ public void testMemoryCapBlocksAdd() throws Exception { public void testConcurrency() throws Exception { Random rand = Randomness.get(); for (int j = 0; j < 5; j++) { // test with different numbers of threads - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); int numThreads = rand.nextInt(50) + 1; ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads); // In this test we want to add the first 200K numbers and check they're all correctly there. @@ -201,7 +205,7 @@ public void testConcurrency() throws Exception { public void testRemoveNoCollisions() throws Exception { long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; int numToAdd = 195000; - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0, memCap); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, memCap); // there should be no collisions for sequential positive numbers up to modulo for (int i = 0; i < numToAdd; i++) { kls.add(i); @@ -217,7 +221,7 @@ public void testRemoveNoCollisions() throws Exception { public void testRemoveWithCollisions() throws Exception { int modulo = (int) Math.pow(2, 26); long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memCap); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap); for (int i = 0; i < 10; i++) { kls.add(i); if (i % 2 == 1) { @@ -260,31 +264,30 @@ public void testRemoveWithCollisions() throws Exception { } public void testNullInputs() throws Exception { - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); assertFalse(kls.add(null)); assertFalse(kls.contains(null)); assertEquals(0, (int) kls.getInternalRepresentation(null)); assertFalse(kls.remove(null)); assertFalse(kls.isCollision(null, null)); - assertEquals(0, kls.getTotalAdds()); - Integer[] newVals = new Integer[]{1, 17, -2, null, -4, null}; + assertEquals(0, kls.getAddAttempts()); + Integer[] newVals = new Integer[] { 1, 17, -2, null, -4, null }; kls.regenerateStore(newVals); assertEquals(4, kls.getSize()); } public void testMemoryCapValueInitialization() { - double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 }; - double[] expectedMultipliers = new double[] { 1.2, 1.2, 1.2, 1, 1, 1 }; - double[] expectedSlopes = new double[] { 0.637, 0.637, 0.637, 0.619, 0.614, 0.629 }; - double[] expectedIntercepts = new double[] { 3.091, 3.091, 3.091, 2.993, 2.905, 2.603 }; + // double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 }; + // 0, 31, 29, 28, 26 + RBMIntKeyLookupStore.KeystoreModuloValue[] mods = RBMIntKeyLookupStore.KeystoreModuloValue.values(); + double[] expectedMultipliers = new double[] { 1.2, 1.2, 1, 1, 1 }; + double[] expectedSlopes = new double[] { 0.637, 0.637, 0.619, 0.614, 0.629 }; + double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 }; // check the numbers closer later long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB; double delta = 0.01; - for (int i = 0; i < logModulos.length; i++) { - int modulo = 0; - if (logModulos[i] != 0) { - modulo = (int) Math.pow(2, logModulos[i]); - } - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes); + for (int i = 0; i < mods.length; i++) { + RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = mods[i]; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes); assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0); assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta); assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta); From 1cf743b25267258c2bc2a335effa7ea8844c3d97 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 6 Nov 2023 13:57:34 -0800 Subject: [PATCH 05/28] Implemented/tested counter+removal list setup to allow more removals Signed-off-by: Peter Alfonsi --- .../tier/keystore/RBMIntKeyLookupStore.java | 91 +++++++++++++-- .../keystore/RBMIntKeyLookupStoreTests.java | 108 +++++++++++++++++- 2 files changed, 184 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index 7e91a64758919..f9696e62a88b7 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -34,6 +34,8 @@ import org.opensearch.common.metrics.CounterMetric; +import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -73,7 +75,8 @@ public int getValue() { protected final int modulo; KeyStoreStats stats; protected RoaringBitmap rbm; - private HashSet collidedInts; + private HashMap collidedIntCounters; + private HashMap> removalSets; protected RBMSizeEstimator sizeEstimator; protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); protected final Lock readLock = lock.readLock(); @@ -89,7 +92,8 @@ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBy sizeEstimator = new RBMSizeEstimator(modulo); this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes)); this.rbm = new RoaringBitmap(); - collidedInts = new HashSet<>(); + this.collidedIntCounters = new HashMap<>(); + this.removalSets = new HashMap<>(); } protected int calculateMaxNumEntries(long memSizeCapInBytes) { @@ -105,7 +109,14 @@ private final int transform(int value) { private void handleCollisions(int transformedValue) { stats.numCollisions.inc(); - collidedInts.add(transformedValue); + CounterMetric numCollisions = collidedIntCounters.get(transformedValue); + if (numCollisions == null) { // First time the transformedValue has had a collision + numCollisions = new CounterMetric(); + numCollisions.inc(2); + collidedIntCounters.put(transformedValue, numCollisions); // Initialize the number of colliding keys to 2 + } else { + numCollisions.inc(); + } } @Override @@ -130,6 +141,16 @@ public boolean add(Integer value) throws Exception { stats.size.inc(); return true; } + // If the value is already pending removal, take it out of the removalList + HashSet removalSet = removalSets.get(transformedValue); + if (removalSet != null) { + removalSet.remove(value); + // Don't increment the counter - this is handled by handleCollisions() later + if (removalSet.isEmpty()) { + removalSets.remove(transformedValue); + } + } + handleCollisions(transformedValue); return false; } finally { @@ -159,6 +180,13 @@ public Integer getInternalRepresentation(Integer value) { return Integer.valueOf(transform(value)); } + /** + * Attempts to remove a value from the keystore. WARNING: Removing keys which have not been added to the keystore + * may cause undefined behavior, including future false negatives!! + * @param value The value to attempt to remove. + * @return true if the value was removed, false otherwise + * @throws Exception + */ @Override public boolean remove(Integer value) throws Exception { if (value == null) { @@ -170,24 +198,54 @@ public boolean remove(Integer value) throws Exception { if (!rbm.contains(transformedValue)) { // saves additional transform() call return false; } + // move below into write lock stats.numRemovalAttempts.inc(); - if (collidedInts.contains(transformedValue)) { - return false; - } } finally { readLock.unlock(); } writeLock.lock(); try { - rbm.remove(transformedValue); - stats.size.dec(); - stats.numSuccessfulRemovals.inc(); + CounterMetric numCollisions = collidedIntCounters.get(transformedValue); + if (numCollisions != null) { + // This transformed value has had a collision before + HashSet removalSet = removalSets.get(transformedValue); + if (removalSet == null) { + // First time a removal has been attempted for this transformed value + HashSet newRemovalSet = new HashSet<>(); + newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this transformedValue + removalSets.put(transformedValue, newRemovalSet); + numCollisions.dec(); + } else { + if (removalSet.contains(value)) { + return false; // We have already attempted to remove this value. Do nothing + } + removalSet.add(value); + numCollisions.dec(); + // If numCollisions has reached zero, we can safely remove all values in removalList + if (numCollisions.count() == 0) { + removeFromRBM(transformedValue); + collidedIntCounters.remove(transformedValue); + removalSets.remove(transformedValue); + return true; + } + } + return false; + } + // Otherwise, there's not been a collision for this transformedValue, so we can safely remove + removeFromRBM(transformedValue); return true; } finally { writeLock.unlock(); } } + // Helper fn for remove() + private void removeFromRBM(int transformedValue) { + rbm.remove(transformedValue); + stats.size.dec(); + stats.numSuccessfulRemovals.inc(); + } + @Override public int getSize() { readLock.lock(); @@ -218,7 +276,7 @@ public boolean isCollision(Integer value1, Integer value2) { @Override public long getMemorySizeInBytes() { - return sizeEstimator.getSizeInBytes((int) stats.size.count()) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); + return sizeEstimator.getSizeInBytes((int) stats.size.count()); // + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); } @Override @@ -234,7 +292,8 @@ public boolean isFull() { @Override public void regenerateStore(Integer[] newValues) throws Exception { rbm.clear(); - collidedInts = new HashSet<>(); + collidedIntCounters = new HashMap<>(); + removalSets = new HashMap<>(); stats.size = new CounterMetric(); stats.numAddAttempts = new CounterMetric(); stats.numCollisions = new CounterMetric(); @@ -265,6 +324,14 @@ public boolean valueHasHadCollision(Integer value) { if (value == null) { return false; } - return collidedInts.contains(transform(value)); + return collidedIntCounters.containsKey(transform(value)); + } + + CounterMetric getNumCollisionsForValue(int value) { // package private for testing + return collidedIntCounters.get(transform(value)); + } + + HashSet getRemovalSetForValue(int value) { + return removalSets.get(transform(value)); } } diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index 08ab08462373b..0e9d63a9731a5 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -34,9 +34,11 @@ import org.opensearch.common.Randomness; import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore; import org.opensearch.common.cache.tier.keystore.RBMSizeEstimator; +import org.opensearch.common.metrics.CounterMetric; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; +import java.util.HashSet; import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -277,12 +279,10 @@ public void testNullInputs() throws Exception { } public void testMemoryCapValueInitialization() { - // double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 }; - // 0, 31, 29, 28, 26 RBMIntKeyLookupStore.KeystoreModuloValue[] mods = RBMIntKeyLookupStore.KeystoreModuloValue.values(); double[] expectedMultipliers = new double[] { 1.2, 1.2, 1, 1, 1 }; double[] expectedSlopes = new double[] { 0.637, 0.637, 0.619, 0.614, 0.629 }; - double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 }; // check the numbers closer later + double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 }; long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB; double delta = 0.01; for (int i = 0; i < mods.length; i++) { @@ -295,4 +295,106 @@ public void testMemoryCapValueInitialization() { } } + + public void testRemovalLogic() throws Exception { + RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX; + int modulo = moduloValue.getValue(); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L); + + // Test standard sequence: add K1, K2, K3 which all transform to C, then: + // Remove K3 + // Remove K2, re-add it, re-remove it twice (duplicate should do nothing) + // Remove K1, which should finally actually remove everything + int c = -42; + int k1 = c + modulo; + int k2 = c + 2 * modulo; + int k3 = c + 3 * modulo; + kls.add(k1); + assertTrue(kls.contains(k1)); + assertTrue(kls.contains(k3)); + kls.add(k2); + CounterMetric numCollisions = kls.getNumCollisionsForValue(k2); + assertNotNull(numCollisions); + assertEquals(2, numCollisions.count()); + kls.add(k3); + assertEquals(3, numCollisions.count()); + assertEquals(1, kls.getSize()); + + boolean removed = kls.remove(k3); + assertFalse(removed); + HashSet removalSet = kls.getRemovalSetForValue(k3); + assertEquals(1, removalSet.size()); + assertTrue(removalSet.contains(k3)); + assertEquals(2, numCollisions.count()); + assertEquals(1, kls.getSize()); + + removed = kls.remove(k2); + assertFalse(removed); + assertEquals(2, removalSet.size()); + assertTrue(removalSet.contains(k2)); + assertEquals(1, numCollisions.count()); + assertEquals(1, kls.getSize()); + + kls.add(k2); + assertEquals(1, removalSet.size()); + assertFalse(removalSet.contains(k2)); + assertEquals(2, numCollisions.count()); + assertEquals(1, kls.getSize()); + + removed = kls.remove(k2); + assertFalse(removed); + assertEquals(2, removalSet.size()); + assertTrue(removalSet.contains(k2)); + assertEquals(1, numCollisions.count()); + assertEquals(1, kls.getSize()); + + removed = kls.remove(k2); + assertFalse(removed); + assertEquals(2, removalSet.size()); + assertTrue(removalSet.contains(k2)); + assertEquals(1, numCollisions.count()); + assertEquals(1, kls.getSize()); + + removed = kls.remove(k1); + assertTrue(removed); + assertNull(kls.getRemovalSetForValue(k1)); + assertNull(kls.getNumCollisionsForValue(k1)); + assertEquals(0, kls.getSize()); + } + + public void testRemovalLogicWithHashCollision() throws Exception { + RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX; + int modulo = moduloValue.getValue(); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, 0L); + + // Test adding K1 twice (maybe two keys hash to K1), then removing it twice. + // We expect it to be unable to remove the last one, but there should be no false negatives. + int c = 77; + int k1 = c + modulo; + int k2 = c + 2 * modulo; + kls.add(k1); + kls.add(k2); + CounterMetric numCollisions = kls.getNumCollisionsForValue(k1); + assertEquals(2, numCollisions.count()); + kls.add(k1); + assertEquals(3, numCollisions.count()); + + boolean removed = kls.remove(k1); + assertFalse(removed); + HashSet removalSet = kls.getRemovalSetForValue(k1); + assertTrue(removalSet.contains(k1)); + assertEquals(2, numCollisions.count()); + + removed = kls.remove(k2); + assertFalse(removed); + assertTrue(removalSet.contains(k2)); + assertEquals(1, numCollisions.count()); + + removed = kls.remove(k1); + assertFalse(removed); + assertTrue(removalSet.contains(k1)); + assertEquals(1, numCollisions.count()); + assertTrue(kls.contains(k1)); + assertTrue(kls.contains(k2)); + } } From cccbdb76af0b50d30460b8a7c2655a4e67b0855a Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 6 Nov 2023 15:20:25 -0800 Subject: [PATCH 06/28] removed exceptions from interface signatures Signed-off-by: Peter Alfonsi --- .../common/cache/tier/keystore/KeyLookupStore.java | 10 +++++----- .../cache/tier/keystore/RBMIntKeyLookupStore.java | 11 +++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java index 552698e0293eb..a08174b5b46d3 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java @@ -47,7 +47,7 @@ public interface KeyLookupStore { * @return true if the value was added, false if it wasn't added because of a * collision or if it was already present. */ - boolean add(T value) throws Exception; + boolean add(T value); /** * Checks if the transformation of the value is in the keystore. @@ -55,7 +55,7 @@ public interface KeyLookupStore { * @return true if the value was found, false otherwise. Due to collisions, false positives are * possible, but there should be no false negatives unless forceRemove() is called. */ - boolean contains(T value) throws Exception; + boolean contains(T value); /** * Returns the transformed version of the input value, that would be used to stored it in the keystore. @@ -72,7 +72,7 @@ public interface KeyLookupStore { * @param value The value to attempt to remove. * @return true if the value was removed, false if it wasn't. */ - boolean remove(T value) throws Exception; + boolean remove(T value); /** * Returns the number of distinct values stored in the internal data structure. @@ -123,10 +123,10 @@ public interface KeyLookupStore { * Also resets all stats related to adding. * @param newValues The keys that should be in the reset structure. */ - void regenerateStore(T[] newValues) throws Exception; + void regenerateStore(T[] newValues); /** * Deletes all keys and resets all stats related to adding. */ - void clear() throws Exception; + void clear(); } diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index f9696e62a88b7..9dba9e8165d40 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -120,7 +120,7 @@ private void handleCollisions(int transformedValue) { } @Override - public boolean add(Integer value) throws Exception { + public boolean add(Integer value) { if (value == null) { return false; } @@ -159,7 +159,7 @@ public boolean add(Integer value) throws Exception { } @Override - public boolean contains(Integer value) throws Exception { + public boolean contains(Integer value) { if (value == null) { return false; } @@ -185,10 +185,9 @@ public Integer getInternalRepresentation(Integer value) { * may cause undefined behavior, including future false negatives!! * @param value The value to attempt to remove. * @return true if the value was removed, false otherwise - * @throws Exception */ @Override - public boolean remove(Integer value) throws Exception { + public boolean remove(Integer value) { if (value == null) { return false; } @@ -290,7 +289,7 @@ public boolean isFull() { } @Override - public void regenerateStore(Integer[] newValues) throws Exception { + public void regenerateStore(Integer[] newValues) { rbm.clear(); collidedIntCounters = new HashMap<>(); removalSets = new HashMap<>(); @@ -308,7 +307,7 @@ public void regenerateStore(Integer[] newValues) throws Exception { } @Override - public void clear() throws Exception { + public void clear() { regenerateStore(new Integer[] {}); } From 14461035b8502ce212e20fe51b964d2c9e171802 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 10 Nov 2023 16:37:14 -0800 Subject: [PATCH 07/28] Simplified RBM size estimator Signed-off-by: Peter Alfonsi --- .../cache/tier/keystore/KeyLookupStore.java | 2 +- .../cache/tier/keystore/KeyStoreStats.java | 4 +- .../tier/keystore/RBMIntKeyLookupStore.java | 41 ++++-- .../cache/tier/keystore/RBMSizeEstimator.java | 137 ------------------ .../keystore/RBMIntKeyLookupStoreTests.java | 64 ++++---- 5 files changed, 63 insertions(+), 185 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java index a08174b5b46d3..dc2b7a4ba1234 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java @@ -103,7 +103,7 @@ public interface KeyLookupStore { /** * Returns an estimate of the store's memory usage. - * @return The memory usage, in MB + * @return The memory usage */ long getMemorySizeInBytes(); diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java index b5c95b134990c..ab3055a81d4c9 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java @@ -22,17 +22,15 @@ public class KeyStoreStats { protected CounterMetric numAddAttempts; protected CounterMetric numCollisions; protected boolean guaranteesNoFalseNegatives; - protected final int maxNumEntries; protected AtomicBoolean atCapacity; protected CounterMetric numRemovalAttempts; protected CounterMetric numSuccessfulRemovals; - protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) { + protected KeyStoreStats(long memSizeCapInBytes) { this.size = new CounterMetric(); this.numAddAttempts = new CounterMetric(); this.numCollisions = new CounterMetric(); this.memSizeCapInBytes = memSizeCapInBytes; - this.maxNumEntries = maxNumEntries; this.atCapacity = new AtomicBoolean(false); this.numRemovalAttempts = new CounterMetric(); this.numSuccessfulRemovals = new CounterMetric(); diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index 9dba9e8165d40..e7370558da80b 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -34,7 +34,6 @@ import org.opensearch.common.metrics.CounterMetric; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.concurrent.locks.Lock; @@ -77,10 +76,14 @@ public int getValue() { protected RoaringBitmap rbm; private HashMap collidedIntCounters; private HashMap> removalSets; - protected RBMSizeEstimator sizeEstimator; protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); protected final Lock readLock = lock.readLock(); protected final Lock writeLock = lock.writeLock(); + private long mostRecentByteEstimate; + private final int REFRESH_SIZE_EST_INTERVAL = 10000; + // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms, + // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store + // in terms of storage impact // Default constructor sets modulo = 2^28 public RBMIntKeyLookupStore(long memSizeCapInBytes) { @@ -89,18 +92,11 @@ public RBMIntKeyLookupStore(long memSizeCapInBytes) { public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) { this.modulo = moduloValue.getValue(); - sizeEstimator = new RBMSizeEstimator(modulo); - this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes)); + this.stats = new KeyStoreStats(memSizeCapInBytes); this.rbm = new RoaringBitmap(); this.collidedIntCounters = new HashMap<>(); this.removalSets = new HashMap<>(); - } - - protected int calculateMaxNumEntries(long memSizeCapInBytes) { - if (memSizeCapInBytes == 0) { - return Integer.MAX_VALUE; - } - return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes); + this.mostRecentByteEstimate = 0L; } private final int transform(int value) { @@ -125,7 +121,11 @@ public boolean add(Integer value) { return false; } stats.numAddAttempts.inc(); - if (stats.size.count() == stats.maxNumEntries) { + + if (getSize() % REFRESH_SIZE_EST_INTERVAL == 0) { + mostRecentByteEstimate = getMemorySizeInBytes(); + } + if (getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes()) { stats.atCapacity.set(true); return false; } @@ -273,9 +273,24 @@ public boolean isCollision(Integer value1, Integer value2) { return transform(value1) == transform(value2); } + static double getRBMSizeMultiplier(int numEntries, int modulo) { + double x = Math.log10((double) numEntries / modulo); + if (x < -5) { + return 7.0; + } + if (x < -2.75) { + return -2.5 * x - 5.5; + } + if (x <= 0) { + return -3.0 / 22.0 * x + 1; + } + return 1; + } + @Override public long getMemorySizeInBytes() { - return sizeEstimator.getSizeInBytes((int) stats.size.count()); // + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); + double multiplier = getRBMSizeMultiplier((int) stats.size.count(), modulo); + return (long) (rbm.getSizeInBytes() * multiplier); } @Override diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java deleted file mode 100644 index e07c645bad0cf..0000000000000 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMSizeEstimator.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.cache.tier.keystore; - -import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore; - -/** - * A class used to estimate roaring bitmap memory sizes (and hash set sizes). - * Values based on experiments with adding randomly distributed integers, which matches the use case for KeyLookupStore. - * In this use case, true values are much higher than an RBM's self-reported size, especially for small RBMs: see - * https://github.com/RoaringBitmap/RoaringBitmap/issues/257 - */ -public class RBMSizeEstimator { - public static final int BYTES_IN_MB = 1048576; - public static final double HASHSET_MEM_SLOPE = 6.46 * Math.pow(10, -5); - protected final double slope; - protected final double bufferMultiplier; - protected final double intercept; - - RBMSizeEstimator(int modulo) { - double[] memValues = calculateMemoryCoefficients(modulo); - this.bufferMultiplier = memValues[0]; - this.slope = memValues[1]; - this.intercept = memValues[2]; - } - - public static double[] calculateMemoryCoefficients(int modulo) { - // Sets up values to help estimate RBM size given a modulo - // Returns an array of {bufferMultiplier, slope, intercept} - - double modifiedModulo; - if (modulo == 0) { - modifiedModulo = 32.0; - } else { - modifiedModulo = Math.log(modulo) / Math.log(2); - } - // we "round up" the modulo to the nearest tested value - double highCutoff = 29.001; // Floating point makes 29 not work - double mediumCutoff = 28.0; - double lowCutoff = 26.0; - double bufferMultiplier = 1.0; - double slope; - double intercept; - if (modifiedModulo > highCutoff) { - // modulo > 2^29 - bufferMultiplier = 1.2; - slope = 0.637; - intercept = 3.091; - } else if (modifiedModulo > mediumCutoff) { - // 2^29 >= modulo > 2^28 - slope = 0.619; - intercept = 2.993; - } else if (modifiedModulo > lowCutoff) { - // 2^28 >= modulo > 2^26 - slope = 0.614; - intercept = 2.905; - } else { - slope = 0.628; - intercept = 2.603; - } - return new double[] { bufferMultiplier, slope, intercept }; - } - - public long getSizeInBytes(int numEntries) { - // Based on a linear fit in log-log space, so that we minimize the error as a proportion rather than as - // an absolute value. Should be within ~50% of the true value at worst, and should overestimate rather - // than underestimate the memory usage - return (long) ((long) Math.pow(numEntries, slope) * (long) Math.pow(10, intercept) * bufferMultiplier); - } - - public int getNumEntriesFromSizeInBytes(long sizeInBytes) { - // This function has some precision issues especially when composed with its inverse: - // numEntries = getNumEntriesFromSizeInBytes(getSizeInBytes(numEntries)) - // In this case the result can be off by up to a couple percent - // However, this shouldn't really matter as both functions are based on memory estimates with higher errors than a couple percent - // and this composition won't happen outside of tests - return (int) Math.pow(sizeInBytes / (bufferMultiplier * Math.pow(10, intercept)), 1 / slope); - - } - - public static long getSizeInBytesWithModulo(int numEntries, int modulo) { - double[] memValues = calculateMemoryCoefficients(modulo); - return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]); - } - - public static long getSizeInBytesWithModuloValue(int numEntries, RBMIntKeyLookupStore.KeystoreModuloValue moduloValue) { - double[] memValues = calculateMemoryCoefficients(moduloValue.getValue()); - return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]); - } - - public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) { - double[] memValues = calculateMemoryCoefficients(modulo); - return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]); - } - - protected static long convertMBToBytes(double valMB) { - return (long) (valMB * BYTES_IN_MB); - } - - protected static double convertBytesToMB(long valBytes) { - return (double) valBytes / BYTES_IN_MB; - } - - protected static long getHashsetMemSizeInBytes(int numEntries) { - return convertMBToBytes(HASHSET_MEM_SLOPE * numEntries); - } -} diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index 0e9d63a9731a5..a6d076c8250ad 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -32,10 +32,9 @@ package org.opensearch.common.cache.tier.keystore; import org.opensearch.common.Randomness; -import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore; -import org.opensearch.common.cache.tier.keystore.RBMSizeEstimator; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.test.OpenSearchTestCase; +import org.roaringbitmap.RoaringBitmap; import java.util.ArrayList; import java.util.HashSet; @@ -45,8 +44,10 @@ import java.util.concurrent.ThreadPoolExecutor; public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase { + + final int BYTES_IN_MB = 1048576; public void testInit() { - long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB; + long memCap = 100 * BYTES_IN_MB; RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap); assertEquals(0, kls.getSize()); assertEquals(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_EIGHT.getValue(), kls.modulo); @@ -129,16 +130,35 @@ public void testAddingDuplicates() throws Exception { } public void testMemoryCapBlocksAdd() throws Exception { + // Now that we're using a modified version of rbm.getSizeInBytes(), which doesn't provide an inverse function, + // we have to test filling just an RBM with random test values first so that we can get the resulting memory cap limit + // to use with our modified size estimate. + // This is much noisier so the precision is lower. + + // It is necessary to use randomly distributed integers for both parts of this test, as we would do with hashes in the cache, + // as that's what our size estimator is designed for. + // If we add a run of integers, our size estimator is not valid, especially for small RBMs. + + int[] maxEntriesArr = new int[] { 1342000, 100000, 3000000}; + long[] rbmReportedSizes = new long[4]; + Random rand = Randomness.get(); + for (int j = 0; j < maxEntriesArr.length; j++) { + RoaringBitmap rbm = new RoaringBitmap(); + for (int i = 0; i < maxEntriesArr[j]; i++) { + rbm.add(rand.nextInt()); + } + rbmReportedSizes[j] = rbm.getSizeInBytes(); + } RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE; - for (int maxEntries : new int[] { 2342000, 1000, 100000 }) { - long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue); + for (int i = 0; i < maxEntriesArr.length; i++) { + double multiplier = RBMIntKeyLookupStore.getRBMSizeMultiplier(maxEntriesArr[i], moduloValue.getValue()); + long memSizeCapInBytes = (long) (rbmReportedSizes[i] * multiplier); + //long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue); RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes); - for (int j = 0; j < maxEntries + 1000; j++) { - kls.add(j); + for (int j = 0; j < maxEntriesArr[i] + 5000; j++) { + kls.add(rand.nextInt()); } - assertTrue(Math.abs(maxEntries - kls.getSize()) < (double) maxEntries / 25); - // exact cap varies a small amount bc of floating point, especially when we use bytes instead of MB for calculations - // precision gets much worse when we compose the two functions, as we do here, but this wouldn't happen in an actual use case + assertTrue(Math.abs(maxEntriesArr[i] - kls.getSize()) < (double) maxEntriesArr[i] / 10); } } @@ -205,7 +225,7 @@ public void testConcurrency() throws Exception { } public void testRemoveNoCollisions() throws Exception { - long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; + long memCap = 100L * BYTES_IN_MB; int numToAdd = 195000; RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, memCap); // there should be no collisions for sequential positive numbers up to modulo @@ -222,7 +242,7 @@ public void testRemoveNoCollisions() throws Exception { public void testRemoveWithCollisions() throws Exception { int modulo = (int) Math.pow(2, 26); - long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; + long memCap = 100L * BYTES_IN_MB; RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX, memCap); for (int i = 0; i < 10; i++) { kls.add(i); @@ -278,24 +298,6 @@ public void testNullInputs() throws Exception { assertEquals(4, kls.getSize()); } - public void testMemoryCapValueInitialization() { - RBMIntKeyLookupStore.KeystoreModuloValue[] mods = RBMIntKeyLookupStore.KeystoreModuloValue.values(); - double[] expectedMultipliers = new double[] { 1.2, 1.2, 1, 1, 1 }; - double[] expectedSlopes = new double[] { 0.637, 0.637, 0.619, 0.614, 0.629 }; - double[] expectedIntercepts = new double[] { 3.091, 3.091, 2.993, 2.905, 2.603 }; - long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB; - double delta = 0.01; - for (int i = 0; i < mods.length; i++) { - RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = mods[i]; - RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes); - assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0); - assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta); - assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta); - assertEquals(expectedIntercepts[i], kls.sizeEstimator.intercept, delta); - } - - } - public void testRemovalLogic() throws Exception { RBMIntKeyLookupStore.KeystoreModuloValue moduloValue = RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_SIX; int modulo = moduloValue.getValue(); @@ -395,6 +397,6 @@ public void testRemovalLogicWithHashCollision() throws Exception { assertTrue(removalSet.contains(k1)); assertEquals(1, numCollisions.count()); assertTrue(kls.contains(k1)); - assertTrue(kls.contains(k2)); + assertTrue(kls.contains(k2)); } } From 0c44af67a44efa8d0f94ff2c49cf604fec377d3b Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Wed, 22 Nov 2023 11:21:33 -0800 Subject: [PATCH 08/28] Optimized modulo calculation with bitmask Signed-off-by: Peter Alfonsi --- .../cache/tier/keystore/RBMIntKeyLookupStore.java | 11 +++++++++-- .../tier/keystore/RBMIntKeyLookupStoreTests.java | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index e7370558da80b..4e56e36c130c0 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -72,6 +72,8 @@ public int getValue() { } protected final int modulo; + protected final int modulo_bitmask; + // Since our modulo is always a power of two we can optimize it by ANDing with a particular bitmask KeyStoreStats stats; protected RoaringBitmap rbm; private HashMap collidedIntCounters; @@ -92,6 +94,11 @@ public RBMIntKeyLookupStore(long memSizeCapInBytes) { public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) { this.modulo = moduloValue.getValue(); + if (modulo > 0) { + this.modulo_bitmask = modulo - 1; // keep last log_2(modulo) bits + } else { + this.modulo_bitmask = -1; // -1 in twos complement is all ones -> includes all bits -> same as no modulo + } this.stats = new KeyStoreStats(memSizeCapInBytes); this.rbm = new RoaringBitmap(); this.collidedIntCounters = new HashMap<>(); @@ -99,8 +106,8 @@ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBy this.mostRecentByteEstimate = 0L; } - private final int transform(int value) { - return modulo == 0 ? value : value % modulo; + private int transform(int value) { + return value & modulo_bitmask; } private void handleCollisions(int transformedValue) { diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index a6d076c8250ad..c0d16dfbc5744 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -61,14 +61,22 @@ public void testTransformationLogic() throws Exception { for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design int posValue = i * modulo + offset; kls.add(posValue); + assertEquals(offset, (int) kls.getInternalRepresentation(posValue)); int negValue = -(i * modulo + offset); kls.add(negValue); + assertEquals(modulo - offset, (int) kls.getInternalRepresentation(negValue)); } assertEquals(2, kls.getSize()); int[] testVals = new int[] { 0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE }; for (int value : testVals) { assertTrue(kls.getInternalRepresentation(value) < modulo); - assertTrue(kls.getInternalRepresentation(value) > -modulo); + assertTrue(kls.getInternalRepresentation(value) >= 0); + } + RBMIntKeyLookupStore no_modulo_kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L); + Random rand = Randomness.get(); + for (int i = 0; i < 100; i++) { + int val = rand.nextInt(); + assertEquals(val, (int) no_modulo_kls.getInternalRepresentation(val)); } } From 88b40d8c8f50a000b00d305e1ec52d8acd36e774 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 27 Nov 2023 11:38:40 -0800 Subject: [PATCH 09/28] Changed memory size estimator to reflect optimized modulo Signed-off-by: Peter Alfonsi --- .../cache/tier/keystore/RBMIntKeyLookupStore.java | 11 +++++++++-- .../tier/keystore/RBMIntKeyLookupStoreTests.java | 6 +++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index 4e56e36c130c0..e690deae7b521 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -82,7 +82,7 @@ public int getValue() { protected final Lock readLock = lock.readLock(); protected final Lock writeLock = lock.writeLock(); private long mostRecentByteEstimate; - private final int REFRESH_SIZE_EST_INTERVAL = 10000; + protected final int REFRESH_SIZE_EST_INTERVAL = 10000; // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms, // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store // in terms of storage impact @@ -281,7 +281,14 @@ public boolean isCollision(Integer value1, Integer value2) { } static double getRBMSizeMultiplier(int numEntries, int modulo) { - double x = Math.log10((double) numEntries / modulo); + double effectiveModulo = (double) modulo / 2; + /* This model was created when we used % operator to calculate modulo. This has range (-modulo, modulo). + Now we have optimized to use a bitmask, which has range [0, modulo). So the number of possible values stored + is halved. */ + if (modulo == 0) { + effectiveModulo = Math.pow(2, 32); + } + double x = Math.log10((double) numEntries / effectiveModulo); if (x < -5) { return 7.0; } diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index c0d16dfbc5744..d9b1ece1310ca 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -82,9 +82,13 @@ public void testTransformationLogic() throws Exception { public void testContains() throws Exception { RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); - for (int i = 0; i < 2000; i++) { + RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L); + for (int i = 0; i < kls.REFRESH_SIZE_EST_INTERVAL + 1000; i++) { + // set upper bound > number of elements to trigger a size check, ensuring we test that too kls.add(i); assertTrue(kls.contains(i)); + noModuloKls.add(i); + assertTrue(noModuloKls.contains(i)); } } From 853448901728ea67a8593530585eba80f6346e8a Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Tue, 9 Jan 2024 12:31:15 -0800 Subject: [PATCH 10/28] Optimized size estimation, cleanup Signed-off-by: Peter Alfonsi --- .../cache/tier/keystore/KeyStoreStats.java | 1 - .../tier/keystore/RBMIntKeyLookupStore.java | 40 ++++++++++++++++--- .../keystore/RBMIntKeyLookupStoreTests.java | 17 ++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java index ab3055a81d4c9..87de1619c2c31 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java @@ -21,7 +21,6 @@ public class KeyStoreStats { protected long memSizeCapInBytes; protected CounterMetric numAddAttempts; protected CounterMetric numCollisions; - protected boolean guaranteesNoFalseNegatives; protected AtomicBoolean atCapacity; protected CounterMetric numRemovalAttempts; protected CounterMetric numSuccessfulRemovals; diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index e690deae7b521..06426c25cd646 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -33,6 +33,9 @@ package org.opensearch.common.cache.tier.keystore; import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.unit.ByteSizeValue; import java.util.HashMap; import java.util.HashSet; @@ -42,7 +45,7 @@ import org.roaringbitmap.RoaringBitmap; /** - * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values. + * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values. * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28. * It also maintains a hash set of values which have had collisions. Values which haven't had collisions can be * safely removed from the store. The fraction of collided values should be low, @@ -82,16 +85,17 @@ public int getValue() { protected final Lock readLock = lock.readLock(); protected final Lock writeLock = lock.writeLock(); private long mostRecentByteEstimate; - protected final int REFRESH_SIZE_EST_INTERVAL = 10000; + static final int REFRESH_SIZE_EST_INTERVAL = 10_000; // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms, // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store // in terms of storage impact - // Default constructor sets modulo = 2^28 + // Use this constructor to specify memory cap with default modulo public RBMIntKeyLookupStore(long memSizeCapInBytes) { this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes); } + // Use this constructor to specify memory cap and modulo public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes) { this.modulo = moduloValue.getValue(); if (modulo > 0) { @@ -130,7 +134,7 @@ public boolean add(Integer value) { stats.numAddAttempts.inc(); if (getSize() % REFRESH_SIZE_EST_INTERVAL == 0) { - mostRecentByteEstimate = getMemorySizeInBytes(); + mostRecentByteEstimate = computeMemorySizeInBytes(); } if (getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes()) { stats.atCapacity.set(true); @@ -280,7 +284,7 @@ public boolean isCollision(Integer value1, Integer value2) { return transform(value1) == transform(value2); } - static double getRBMSizeMultiplier(int numEntries, int modulo) { + static double getRBMSizeMultiplier(int numEntries, int modulo) { double effectiveModulo = (double) modulo / 2; /* This model was created when we used % operator to calculate modulo. This has range (-modulo, modulo). Now we have optimized to use a bitmask, which has range [0, modulo). So the number of possible values stored @@ -301,8 +305,20 @@ static double getRBMSizeMultiplier(int numEntries, int modulo) { return 1; } + /** + * Return the most recent memory size estimate, without updating it. + * @return the size estimate (bytes) + */ @Override public long getMemorySizeInBytes() { + return mostRecentByteEstimate; + } + + /** + * Calculate a new memory size estimate. This is somewhat expensive, so we don't call this every time we run get(). + * @return a new size estimate (bytes) + */ + private long computeMemorySizeInBytes() { double multiplier = getRBMSizeMultiplier((int) stats.size.count(), modulo); return (long) (rbm.getSizeInBytes() * multiplier); } @@ -325,7 +341,6 @@ public void regenerateStore(Integer[] newValues) { stats.size = new CounterMetric(); stats.numAddAttempts = new CounterMetric(); stats.numCollisions = new CounterMetric(); - stats.guaranteesNoFalseNegatives = true; stats.numRemovalAttempts = new CounterMetric(); stats.numSuccessfulRemovals = new CounterMetric(); for (int i = 0; i < newValues.length; i++) { @@ -362,4 +377,17 @@ CounterMetric getNumCollisionsForValue(int value) { // package private for testi HashSet getRemovalSetForValue(int value) { return removalSets.get(transform(value)); } + + /** + * Function to set a new memory size cap. + * TODO: Integrate this with the tiered caching cluster settings PR once this is raised. + * @param newMemSizeCap The new cap size. + */ + protected void setMemSizeCap(ByteSizeValue newMemSizeCap) { + stats.memSizeCapInBytes = newMemSizeCap.getBytes(); + mostRecentByteEstimate = getMemorySizeInBytes(); + if (mostRecentByteEstimate > getMemorySizeCapInBytes()) { + stats.atCapacity.set(true); + } + } } diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index d9b1ece1310ca..830700a2745df 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -33,6 +33,8 @@ import org.opensearch.common.Randomness; import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.test.OpenSearchTestCase; import org.roaringbitmap.RoaringBitmap; @@ -411,4 +413,19 @@ public void testRemovalLogicWithHashCollision() throws Exception { assertTrue(kls.contains(k1)); assertTrue(kls.contains(k2)); } + + public void testSetMemSizeCap() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0L); // no memory cap + Random rand = Randomness.get(); + for (int i = 0; i < RBMIntKeyLookupStore.REFRESH_SIZE_EST_INTERVAL * 3; i++) { + kls.add(rand.nextInt()); + } + long memSize = kls.getMemorySizeInBytes(); + assertEquals(0, kls.getMemorySizeCapInBytes()); + kls.setMemSizeCap(new ByteSizeValue(memSize / 2, ByteSizeUnit.BYTES)); + // check the keystore is now full and has its lower cap + assertTrue(kls.isFull()); + assertEquals(memSize / 2, kls.getMemorySizeCapInBytes()); + assertFalse(kls.add(rand.nextInt())); + } } From 65d295551a36a89733047040b597b5ec762654d7 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Tue, 9 Jan 2024 13:43:39 -0800 Subject: [PATCH 11/28] Addressed Kiran's comments Signed-off-by: Peter Alfonsi --- .../tier/keystore/RBMIntKeyLookupStore.java | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java index 06426c25cd646..63ce12d17ba93 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java @@ -85,12 +85,15 @@ public int getValue() { protected final Lock readLock = lock.readLock(); protected final Lock writeLock = lock.writeLock(); private long mostRecentByteEstimate; - static final int REFRESH_SIZE_EST_INTERVAL = 10_000; + // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms, // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store // in terms of storage impact + static final int REFRESH_SIZE_EST_INTERVAL = 10_000; - // Use this constructor to specify memory cap with default modulo + + // Use this constructor to specify memory cap with default modulo = 2^28, which we found in experiments + // to be the best tradeoff between lower memory usage and risk of collisions public RBMIntKeyLookupStore(long memSizeCapInBytes) { this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes); } @@ -119,13 +122,21 @@ private void handleCollisions(int transformedValue) { CounterMetric numCollisions = collidedIntCounters.get(transformedValue); if (numCollisions == null) { // First time the transformedValue has had a collision numCollisions = new CounterMetric(); - numCollisions.inc(2); - collidedIntCounters.put(transformedValue, numCollisions); // Initialize the number of colliding keys to 2 + numCollisions.inc(2); // initialize to 2, since the first collision means 2 keys have collided + collidedIntCounters.put(transformedValue, numCollisions); } else { numCollisions.inc(); } } + private boolean shouldUpdateByteEstimate() { + return getSize() % REFRESH_SIZE_EST_INTERVAL == 0; + } + + private boolean isAtCapacityLimit() { + return getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes(); + } + @Override public boolean add(Integer value) { if (value == null) { @@ -133,10 +144,10 @@ public boolean add(Integer value) { } stats.numAddAttempts.inc(); - if (getSize() % REFRESH_SIZE_EST_INTERVAL == 0) { + if (shouldUpdateByteEstimate()) { mostRecentByteEstimate = computeMemorySizeInBytes(); } - if (getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes()) { + if (isAtCapacityLimit()) { stats.atCapacity.set(true); return false; } @@ -144,10 +155,7 @@ public boolean add(Integer value) { writeLock.lock(); try { - boolean alreadyContained; - // saves calling transform() an additional time - alreadyContained = rbm.contains(transformedValue); - if (!alreadyContained) { + if (!rbm.contains(transformedValue)) { rbm.add(transformedValue); stats.size.inc(); return true; @@ -208,7 +216,6 @@ public boolean remove(Integer value) { if (!rbm.contains(transformedValue)) { // saves additional transform() call return false; } - // move below into write lock stats.numRemovalAttempts.inc(); } finally { readLock.unlock(); @@ -251,6 +258,9 @@ public boolean remove(Integer value) { // Helper fn for remove() private void removeFromRBM(int transformedValue) { + if (!lock.isWriteLockedByCurrentThread()) { + throw new IllegalStateException("Write Lock must be held when calling this method"); + } rbm.remove(transformedValue); stats.size.dec(); stats.numSuccessfulRemovals.inc(); @@ -284,6 +294,12 @@ public boolean isCollision(Integer value1, Integer value2) { return transform(value1) == transform(value2); } + /* + The built-in RBM size estimator is known to work very badly for randomly-distributed data, like the hashes we will be using. + See https://github.com/RoaringBitmap/RoaringBitmap/issues/257. + We ran tests to determine what multiplier you need to get true size from reported size, as a function of log10(# entries / modulo), + and found this piecewise linear function was a good approximation across different modulos. + */ static double getRBMSizeMultiplier(int numEntries, int modulo) { double effectiveModulo = (double) modulo / 2; /* This model was created when we used % operator to calculate modulo. This has range (-modulo, modulo). From 9790c16df5256ebc599410fca20ceb14169cb8d3 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Wed, 17 Jan 2024 13:15:57 -0800 Subject: [PATCH 12/28] Fixed static variable warning Signed-off-by: Peter Alfonsi --- .../common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java index 830700a2745df..7626caa14a21f 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java @@ -85,7 +85,7 @@ public void testTransformationLogic() throws Exception { public void testContains() throws Exception { RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.TWO_TO_TWENTY_NINE, 0L); RBMIntKeyLookupStore noModuloKls = new RBMIntKeyLookupStore(RBMIntKeyLookupStore.KeystoreModuloValue.NONE, 0L); - for (int i = 0; i < kls.REFRESH_SIZE_EST_INTERVAL + 1000; i++) { + for (int i = 0; i < RBMIntKeyLookupStore.REFRESH_SIZE_EST_INTERVAL + 1000; i++) { // set upper bound > number of elements to trigger a size check, ensuring we test that too kls.add(i); assertTrue(kls.contains(i)); From 30ad4e9b10ce473286ec5a6ab591fa97dfdbe876 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Mon, 22 Jan 2024 09:42:12 -0800 Subject: [PATCH 13/28] Fixing gradle build failure Signed-off-by: Sagar Upadhyaya --- modules/transport-netty4/build.gradle | 7 ------- plugins/transport-nio/build.gradle | 7 ------- plugins/transport-reactor-netty4/build.gradle | 7 ------- .../java/org/opensearch/common/cache/stats/CacheStats.java | 2 +- 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 83c4db80b7798..dd19650b8da9e 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -177,13 +177,6 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional - 'org.slf4j.helpers.FormattingTuple', - 'org.slf4j.helpers.MessageFormatter', - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory', - 'org.slf4j.spi.LocationAwareLogger', - 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index 8c0ee8ba718ac..4e503413c265b 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -103,13 +103,6 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional - 'org.slf4j.helpers.FormattingTuple', - 'org.slf4j.helpers.MessageFormatter', - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory', - 'org.slf4j.spi.LocationAwareLogger', - 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/plugins/transport-reactor-netty4/build.gradle b/plugins/transport-reactor-netty4/build.gradle index 7d7eb330b4a55..74ceb2ab5687a 100644 --- a/plugins/transport-reactor-netty4/build.gradle +++ b/plugins/transport-reactor-netty4/build.gradle @@ -136,13 +136,6 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', - // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional - 'org.slf4j.helpers.FormattingTuple', - 'org.slf4j.helpers.MessageFormatter', - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory', - 'org.slf4j.spi.LocationAwareLogger', - 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java index a952a2485ed23..cf84f296916fb 100644 --- a/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java +++ b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java @@ -13,6 +13,6 @@ * TODO: Add rest of stats like hits/misses. */ public interface CacheStats { - // Provides the number of entries in cache. + // Provides the current number of entries in cache. long count(); } From 6bcf2f4f8b36a96a7393989be63c3c9051017ae3 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Mon, 22 Jan 2024 09:51:56 -0800 Subject: [PATCH 14/28] Adding changelog Signed-off-by: Sagar Upadhyaya --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10338f6646053..3cce25d971bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,6 +125,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce cluster level setting `cluster.index.restrict.replication.type` to prevent replication type setting override during index creations([#11583](https://github.com/opensearch-project/OpenSearch/pull/11583)) - Add match_only_text field that is optimized for storage by trading off positional queries performance ([#6836](https://github.com/opensearch-project/OpenSearch/pull/11039)) - Introduce new feature flag "WRITEABLE_REMOTE_INDEX" to gate the writeable remote index functionality ([#11717](https://github.com/opensearch-project/OpenSearch/pull/11170)) +- [Tiered caching] Integrating ehcache as a disk cache option ([#11874](https://github.com/opensearch-project/OpenSearch/pull/11874)) ### Dependencies - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822)) From a5e5afdae51328a5833fc5f281ee354c88437d6c Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Thu, 25 Jan 2024 13:12:54 -0800 Subject: [PATCH 15/28] Exposing ehcache disk cache variant as a plugin Signed-off-by: Sagar Upadhyaya --- modules/transport-netty4/build.gradle | 7 + plugins/cache-ehcache/.gitignore | 29 ++ plugins/cache-ehcache/build.gradle | 81 +++++ .../licenses/ehcache-3.10.8.jar.sha1 | 0 .../licenses/ehcache-LICENSE.txt | 0 .../licenses/ehcache-NOTICE.txt | 0 .../opensearch/cache/EhcacheCachePlugin.java | 39 +++ .../opensearch/cache/EhcacheDiskCache.java | 324 +++++++++++------- .../org/opensearch/cache/EhcacheSettings.java | 52 +++ .../org/opensearch/cache/package-info.java | 10 + .../plugin-metadata/plugin-security.policy | 15 + .../cache}/EhCacheDiskCacheTests.java | 91 +++-- .../opensearch/cache/EhcachePluginTests.java | 26 ++ .../licenses/slf4j-api-1.7.36.jar.sha1 | 0 .../licenses/slf4j-api-LICENSE.txt | 0 .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + .../licenses/slf4j-api-LICENSE.txt | 21 ++ .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + .../licenses/slf4j-api-LICENSE.txt | 21 ++ .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + .../licenses/slf4j-api-LICENSE.txt | 21 ++ .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + .../licenses/slf4j-api-LICENSE.txt | 21 ++ .../licenses/slf4j-api-NOTICE.txt | 0 .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + .../licenses/slf4j-api-LICENSE.txt | 21 ++ .../licenses/slf4j-api-NOTICE.txt | 0 plugins/transport-nio/build.gradle | 7 + plugins/transport-reactor-netty4/build.gradle | 7 + server/build.gradle | 3 - .../common/cache/provider/CacheProvider.java | 91 +++++ .../common/cache/provider/package-info.java | 10 + .../common/cache/store/StoreAwareCache.java | 12 + .../builders/StoreAwareCacheBuilder.java | 14 +- .../store/config/StoreAwareCacheConfig.java | 195 +++++++++++ .../cache/store/config/package-info.java | 10 + .../cache/store/enums/CacheStoreType.java | 3 + .../main/java/org/opensearch/node/Node.java | 3 + .../org/opensearch/plugins/CachePlugin.java | 28 ++ 43 files changed, 986 insertions(+), 181 deletions(-) create mode 100644 plugins/cache-ehcache/.gitignore create mode 100644 plugins/cache-ehcache/build.gradle rename {server => plugins/cache-ehcache}/licenses/ehcache-3.10.8.jar.sha1 (100%) rename {server => plugins/cache-ehcache}/licenses/ehcache-LICENSE.txt (100%) rename {server => plugins/cache-ehcache}/licenses/ehcache-NOTICE.txt (100%) create mode 100644 plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheCachePlugin.java rename server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java => plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java (72%) create mode 100644 plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java create mode 100644 plugins/cache-ehcache/src/main/java/org/opensearch/cache/package-info.java create mode 100644 plugins/cache-ehcache/src/main/plugin-metadata/plugin-security.policy rename {server/src/test/java/org/opensearch/common/cache/store => plugins/cache-ehcache/src/test/java/org/opensearch/cache}/EhCacheDiskCacheTests.java (93%) create mode 100644 plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhcachePluginTests.java rename {server => plugins/discovery-ec2}/licenses/slf4j-api-1.7.36.jar.sha1 (100%) rename {server => plugins/discovery-ec2}/licenses/slf4j-api-LICENSE.txt (100%) rename {server => plugins/discovery-ec2}/licenses/slf4j-api-NOTICE.txt (100%) create mode 100644 plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt create mode 100644 plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt create mode 100644 plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt create mode 100644 plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt create mode 100644 plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 plugins/repository-azure/licenses/slf4j-api-LICENSE.txt create mode 100644 plugins/repository-azure/licenses/slf4j-api-NOTICE.txt create mode 100644 plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt create mode 100644 plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt create mode 100644 plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 plugins/repository-s3/licenses/slf4j-api-LICENSE.txt create mode 100644 plugins/repository-s3/licenses/slf4j-api-NOTICE.txt create mode 100644 server/src/main/java/org/opensearch/common/cache/provider/CacheProvider.java create mode 100644 server/src/main/java/org/opensearch/common/cache/provider/package-info.java create mode 100644 server/src/main/java/org/opensearch/common/cache/store/config/StoreAwareCacheConfig.java create mode 100644 server/src/main/java/org/opensearch/common/cache/store/config/package-info.java create mode 100644 server/src/main/java/org/opensearch/plugins/CachePlugin.java diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index dd19650b8da9e..83c4db80b7798 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -177,6 +177,13 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', + // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional + 'org.slf4j.helpers.FormattingTuple', + 'org.slf4j.helpers.MessageFormatter', + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + 'org.slf4j.spi.LocationAwareLogger', + 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/plugins/cache-ehcache/.gitignore b/plugins/cache-ehcache/.gitignore new file mode 100644 index 0000000000000..f68d1099657e3 --- /dev/null +++ b/plugins/cache-ehcache/.gitignore @@ -0,0 +1,29 @@ +### IntelliJ IDEA ### +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/plugins/cache-ehcache/build.gradle b/plugins/cache-ehcache/build.gradle new file mode 100644 index 0000000000000..87c38f616cbb3 --- /dev/null +++ b/plugins/cache-ehcache/build.gradle @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.apache.tools.ant.taskdefs.condition.Os +import org.opensearch.gradle.Architecture +import org.opensearch.gradle.OS +import org.opensearch.gradle.info.BuildParams + +opensearchplugin { + description 'Ehcache based cache implementation.' + classname 'org.opensearch.cache.EhcacheCachePlugin' + hasClientJar = true +} + +dependencies { + api "org.ehcache:ehcache:${versions.ehcache}" +} + +thirdPartyAudit { + ignoreViolations( + 'org.ehcache.impl.internal.concurrent.ConcurrentHashMap', + 'org.ehcache.impl.internal.concurrent.ConcurrentHashMap$CounterCell', + 'org.ehcache.impl.internal.concurrent.ConcurrentHashMap$TreeBin', + 'org.ehcache.impl.internal.concurrent.ThreadLocalRandomUtil', + 'org.ehcache.sizeof.impl.UnsafeSizeOf' + ) + + ignoreMissingClasses( + 'javax.cache.Cache', + 'javax.cache.Cache$Entry', + 'javax.cache.CacheException', + 'javax.cache.CacheManager', + 'javax.cache.configuration.CacheEntryListenerConfiguration', + 'javax.cache.configuration.CompleteConfiguration', + 'javax.cache.configuration.Configuration', + 'javax.cache.configuration.Factory', + 'javax.cache.configuration.OptionalFeature', + 'javax.cache.event.CacheEntryCreatedListener', + 'javax.cache.event.CacheEntryEvent', + 'javax.cache.event.CacheEntryEventFilter', + 'javax.cache.event.CacheEntryExpiredListener', + 'javax.cache.event.CacheEntryListener', + 'javax.cache.event.CacheEntryRemovedListener', + 'javax.cache.event.CacheEntryUpdatedListener', + 'javax.cache.event.EventType', + 'javax.cache.expiry.Duration', + 'javax.cache.expiry.EternalExpiryPolicy', + 'javax.cache.expiry.ExpiryPolicy', + 'javax.cache.integration.CacheLoader', + 'javax.cache.integration.CacheLoaderException', + 'javax.cache.integration.CacheWriter', + 'javax.cache.integration.CacheWriterException', + 'javax.cache.integration.CompletionListener', + 'javax.cache.management.CacheMXBean', + 'javax.cache.management.CacheStatisticsMXBean', + 'javax.cache.processor.EntryProcessor', + 'javax.cache.processor.EntryProcessorResult', + 'javax.cache.processor.MutableEntry', + 'javax.cache.spi.CachingProvider', + 'javax.xml.bind.JAXBContext', + 'javax.xml.bind.JAXBElement', + 'javax.xml.bind.Marshaller', + 'javax.xml.bind.Unmarshaller', + 'javax.xml.bind.annotation.XmlElement', + 'javax.xml.bind.annotation.XmlRootElement', + 'javax.xml.bind.annotation.XmlSchema', + 'javax.xml.bind.annotation.adapters.XmlAdapter', + 'org.osgi.framework.BundleActivator', + 'org.osgi.framework.BundleContext', + 'org.osgi.framework.ServiceReference', + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + 'org.slf4j.Marker', + 'org.slf4j.event.Level' + ) +} diff --git a/server/licenses/ehcache-3.10.8.jar.sha1 b/plugins/cache-ehcache/licenses/ehcache-3.10.8.jar.sha1 similarity index 100% rename from server/licenses/ehcache-3.10.8.jar.sha1 rename to plugins/cache-ehcache/licenses/ehcache-3.10.8.jar.sha1 diff --git a/server/licenses/ehcache-LICENSE.txt b/plugins/cache-ehcache/licenses/ehcache-LICENSE.txt similarity index 100% rename from server/licenses/ehcache-LICENSE.txt rename to plugins/cache-ehcache/licenses/ehcache-LICENSE.txt diff --git a/server/licenses/ehcache-NOTICE.txt b/plugins/cache-ehcache/licenses/ehcache-NOTICE.txt similarity index 100% rename from server/licenses/ehcache-NOTICE.txt rename to plugins/cache-ehcache/licenses/ehcache-NOTICE.txt diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheCachePlugin.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheCachePlugin.java new file mode 100644 index 0000000000000..faf0b5ca79f04 --- /dev/null +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheCachePlugin.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cache; + +import org.opensearch.common.cache.store.StoreAwareCache; +import org.opensearch.common.cache.store.enums.CacheStoreType; +import org.opensearch.plugins.CachePlugin; +import org.opensearch.plugins.Plugin; + +import java.util.Map; + +/** + * Ehcache based cache plugin. + */ +public class EhcacheCachePlugin extends Plugin implements CachePlugin { + + private static final String EHCACHE_CACHE_PLUGIN = "EhcachePlugin"; + + /** + * Default constructor to avoid javadoc related failures. + */ + public EhcacheCachePlugin() {} + + @Override + public Map getCacheStoreTypeMap() { + return Map.of(CacheStoreType.DISK, new EhcacheDiskCache.EhcacheDiskCacheFactory()); + } + + @Override + public String getName() { + return EHCACHE_CACHE_PLUGIN; + } +} diff --git a/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java similarity index 72% rename from server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java rename to plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java index 80def5f260cc5..8ecf38169fe8e 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/EhCacheDiskCache.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java @@ -6,21 +6,25 @@ * compatible open source license. */ -package org.opensearch.common.cache.store; +package org.opensearch.cache; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.cache.LoadAwareCacheLoader; import org.opensearch.common.cache.RemovalReason; +import org.opensearch.common.cache.provider.CacheProvider; import org.opensearch.common.cache.stats.CacheStats; +import org.opensearch.common.cache.store.StoreAwareCache; +import org.opensearch.common.cache.store.StoreAwareCacheRemovalNotification; import org.opensearch.common.cache.store.builders.StoreAwareCacheBuilder; +import org.opensearch.common.cache.store.config.StoreAwareCacheConfig; import org.opensearch.common.cache.store.enums.CacheStoreType; import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; import org.opensearch.common.collect.Tuple; import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -53,6 +57,11 @@ import org.ehcache.spi.loaderwriter.CacheLoadingException; import org.ehcache.spi.loaderwriter.CacheWritingException; +import static org.opensearch.cache.EhcacheSettings.DISK_SEGMENTS; +import static org.opensearch.cache.EhcacheSettings.DISK_WRITE_CONCURRENCY; +import static org.opensearch.cache.EhcacheSettings.DISK_WRITE_MAXIMUM_THREADS; +import static org.opensearch.cache.EhcacheSettings.DISK_WRITE_MINIMUM_THREADS; + /** * This variant of disk cache uses Ehcache underneath. * @param Type of key. @@ -61,9 +70,10 @@ * @opensearch.experimental * */ -public class EhCacheDiskCache implements StoreAwareCache { +@ExperimentalApi +public class EhcacheDiskCache implements StoreAwareCache { - private static final Logger logger = LogManager.getLogger(EhCacheDiskCache.class); + private static final Logger logger = LogManager.getLogger(EhcacheDiskCache.class); // A Cache manager can create many caches. private final PersistentCacheManager cacheManager; @@ -93,22 +103,6 @@ public class EhCacheDiskCache implements StoreAwareCache { private final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB - // Ehcache disk write minimum threads for its pool - public final Setting DISK_WRITE_MINIMUM_THREADS; - - // Ehcache disk write maximum threads for its pool - public final Setting DISK_WRITE_MAXIMUM_THREADS; - - // Not be to confused with number of disk segments, this is different. Defines - // distinct write queues created for disk store where a group of segments share a write queue. This is - // implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write - // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool. - public final Setting DISK_WRITE_CONCURRENCY; - - // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but - // will hold that many file pointers. Default is 16. - public final Setting DISK_SEGMENTS; - private final StoreAwareCacheEventListener eventListener; /** @@ -117,7 +111,7 @@ public class EhCacheDiskCache implements StoreAwareCache { */ Map>> completableFutureMap = new ConcurrentHashMap<>(); - private EhCacheDiskCache(Builder builder) { + private EhcacheDiskCache(Builder builder) { this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null"); this.valueType = Objects.requireNonNull(builder.valueType, "Value type shouldn't be null"); this.expireAfterAccess = Objects.requireNonNull(builder.getExpireAfterAcess(), "ExpireAfterAccess value shouldn't " + "be null"); @@ -131,14 +125,7 @@ private EhCacheDiskCache(Builder builder) { } else { this.threadPoolAlias = builder.threadPoolAlias; } - this.settings = Objects.requireNonNull(builder.settings, "Settings objects shouldn't be null"); - Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null"); - this.DISK_WRITE_MINIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tier.disk.ehcache.min_threads", 2, 1, 5); - this.DISK_WRITE_MAXIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tier.disk.ehcache.max_threads", 2, 1, 20); - // Default value is 1 within EhCache. - this.DISK_WRITE_CONCURRENCY = Setting.intSetting(builder.settingPrefix + ".tier.disk.ehcache.concurrency", 2, 1, 3); - // Default value is 16 within Ehcache. - this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + "tier.disk.ehcache.segments", 16, 1, 32); + this.settings = Objects.requireNonNull(builder.getSettings(), "Settings objects shouldn't be null"); this.cacheManager = buildCacheManager(); Objects.requireNonNull(builder.getEventListener(), "Listener can't be null"); this.eventListener = builder.getEventListener(); @@ -146,53 +133,46 @@ private EhCacheDiskCache(Builder builder) { this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder); } - @SuppressForbidden(reason = "Ehcache uses File.io") - private PersistentCacheManager buildCacheManager() { - // In case we use multiple ehCaches, we can define this cache manager at a global level. - return CacheManagerBuilder.newCacheManagerBuilder() - .with(CacheManagerBuilder.persistence(new File(storagePath))) - .using( - PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder() - .defaultPool(THREAD_POOL_ALIAS_PREFIX + "Default", 1, 3) // Default pool used for other tasks like - // event listeners - .pool(this.threadPoolAlias, DISK_WRITE_MINIMUM_THREADS.get(settings), DISK_WRITE_MAXIMUM_THREADS.get(settings)) - .build() - ) - .build(true); - } - private Cache buildCache(Duration expireAfterAccess, Builder builder) { - return this.cacheManager.createCache( - DISK_CACHE_ALIAS, - CacheConfigurationBuilder.newCacheConfigurationBuilder( - this.keyType, - this.valueType, - ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B) - ).withExpiry(new ExpiryPolicy<>() { - @Override - public Duration getExpiryForCreation(K key, V value) { - return INFINITE; - } - - @Override - public Duration getExpiryForAccess(K key, Supplier value) { - return expireAfterAccess; - } - - @Override - public Duration getExpiryForUpdate(K key, Supplier oldValue, V newValue) { - return INFINITE; - } - }) - .withService(getListenerConfiguration(builder)) - .withService( - new OffHeapDiskStoreConfiguration( - this.threadPoolAlias, - DISK_WRITE_CONCURRENCY.get(settings), - DISK_SEGMENTS.get(settings) + try { + return this.cacheManager.createCache( + builder.diskCacheAlias, + CacheConfigurationBuilder.newCacheConfigurationBuilder( + this.keyType, + this.valueType, + ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B) + ).withExpiry(new ExpiryPolicy<>() { + @Override + public Duration getExpiryForCreation(K key, V value) { + return INFINITE; + } + + @Override + public Duration getExpiryForAccess(K key, Supplier value) { + return expireAfterAccess; + } + + @Override + public Duration getExpiryForUpdate(K key, Supplier oldValue, V newValue) { + return INFINITE; + } + }) + .withService(getListenerConfiguration(builder)) + .withService( + new OffHeapDiskStoreConfiguration( + this.threadPoolAlias, + DISK_WRITE_CONCURRENCY.get(settings), + DISK_SEGMENTS.get(settings) + ) ) - ) - ); + ); + } catch (IllegalArgumentException ex) { + logger.error("Ehcache disk cache initialization failed due to illegal argument: {}", ex.getMessage()); + throw ex; + } catch (IllegalStateException ex) { + logger.error("Ehcache disk cache initialization failed: {}", ex.getMessage()); + throw ex; + } } private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder builder) { @@ -216,12 +196,26 @@ Map>> getCompletableFutureMap() { return completableFutureMap; } + @SuppressForbidden(reason = "Ehcache uses File.io") + private PersistentCacheManager buildCacheManager() { + // In case we use multiple ehCaches, we can define this cache manager at a global level. + return CacheManagerBuilder.newCacheManagerBuilder() + .with(CacheManagerBuilder.persistence(new File(storagePath))) + .using( + PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder() + .defaultPool(THREAD_POOL_ALIAS_PREFIX + "Default", 1, 3) // Default pool used for other tasks like + // event listeners + .pool(this.threadPoolAlias, DISK_WRITE_MINIMUM_THREADS.get(settings), DISK_WRITE_MAXIMUM_THREADS.get(settings)) + .build() + ) + .build(true); + } + @Override public V get(K key) { if (key == null) { throw new IllegalArgumentException("Key passed to ehcache disk cache was null."); } - // Optimize it by adding key store. V value; try { value = cache.get(key); @@ -236,6 +230,11 @@ public V get(K key) { return value; } + /** + * Puts the item into cache. + * @param key Type of key. + * @param value Type of value. + */ @Override public void put(K key, V value) { try { @@ -245,6 +244,13 @@ public void put(K key, V value) { } } + /** + * Computes the value using loader in case key is not present, otherwise fetches it. + * @param key Type of key + * @param loader loader to load the value in case key is missing + * @return value + * @throws Exception + */ @Override public V computeIfAbsent(K key, LoadAwareCacheLoader loader) throws Exception { // Ehache doesn't provide any computeIfAbsent function. Exposes putIfAbsent but that works differently and is @@ -315,27 +321,37 @@ private V compute(K key, LoadAwareCacheLoader loader) throws Exception { return value; } + /** + * Invalidate the item. + * @param key key to be invalidated. + */ @Override public void invalidate(K key) { - // There seems to be an thread leak issue while calling this and then closing cache. try { cache.remove(key); } catch (CacheWritingException ex) { // Handle throw new RuntimeException(ex); } + } @Override - public void invalidateAll() { - // TODO - } + public void invalidateAll() {} + /** + * Provides a way to iterate over disk cache keys. + * @return Iterable + */ @Override public Iterable keys() { return () -> new EhCacheKeyIterator<>(cache.iterator()); } + /** + * Gives the current count of keys in disk cache. + * @return + */ @Override public long count() { return stats.count(); @@ -346,11 +362,6 @@ public void refresh() { // TODO: ehcache doesn't provide a way to refresh a cache. } - @Override - public CacheStoreType getTierType() { - return CacheStoreType.DISK; - } - @Override public void close() { cacheManager.removeCache(DISK_CACHE_ALIAS); @@ -362,16 +373,29 @@ public void close() { } } + /** + * Relevant stats for this cache. + * @return CacheStats + */ @Override public CacheStats stats() { return stats; } + /** + * Returns the tier type. + * @return CacheStoreType.DISK + */ + @Override + public CacheStoreType getTierType() { + return CacheStoreType.DISK; + } + /** * Stats related to disk cache. */ - class DiskCacheStats implements CacheStats { - private CounterMetric count = new CounterMetric(); + static class DiskCacheStats implements CacheStats { + private final CounterMetric count = new CounterMetric(); @Override public long count() { @@ -379,6 +403,32 @@ public long count() { } } + /** + * This iterator wraps ehCache iterator and only iterates over its keys. + * @param Type of key + */ + class EhCacheKeyIterator implements Iterator { + + Iterator> iterator; + + EhCacheKeyIterator(Iterator> iterator) { + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public K next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return iterator.next().getKey(); + } + } + /** * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers. * @param Type of key @@ -445,28 +495,26 @@ public void onEvent(CacheEvent event) { } /** - * This iterator wraps ehCache iterator and only iterates over its keys. - * @param Type of key + * Factory to create an ehcache disk cache. */ - class EhCacheKeyIterator implements Iterator { - - Iterator> iterator; - - EhCacheKeyIterator(Iterator> iterator) { - this.iterator = iterator; - } + static class EhcacheDiskCacheFactory implements StoreAwareCache.Factory { @Override - public boolean hasNext() { - return iterator.hasNext(); + public StoreAwareCache create(StoreAwareCacheConfig storeAwareCacheConfig) { + return new Builder().setStoragePath(storeAwareCacheConfig.getStoragePath()) + .setDiskCacheAlias(storeAwareCacheConfig.getCacheAlias()) + .setKeyType(storeAwareCacheConfig.getKeyType()) + .setValueType(storeAwareCacheConfig.getValueType()) + .setEventListener(storeAwareCacheConfig.getEventListener()) + .setExpireAfterAccess(storeAwareCacheConfig.getExpireAfterAcess()) + .setMaximumWeightInBytes(storeAwareCacheConfig.getMaxWeightInBytes()) + .setSettings(storeAwareCacheConfig.getSettings()) + .build(); } @Override - public K next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - return iterator.next().getKey(); + public String getCacheName() { + return CacheProvider.CacheType.EHCACHE.getValue(); } } @@ -476,68 +524,88 @@ public K next() { * @param Type of value */ public static class Builder extends StoreAwareCacheBuilder { - private Class keyType; - - private Class valueType; private String storagePath; private String threadPoolAlias; - private Settings settings; - private String diskCacheAlias; - private String settingPrefix; - // Provides capability to make ehCache event listener to run in sync mode. Used for testing too. private boolean isEventListenerModeSync; + private Class keyType; + + private Class valueType; + + /** + * Default constructor. Added to fix javadocs. + */ public Builder() {} - public EhCacheDiskCache.Builder setKeyType(Class keyType) { + /** + * Sets the key type of value. + * @param keyType type of key + * @return builder + */ + public Builder setKeyType(Class keyType) { this.keyType = keyType; return this; } - public EhCacheDiskCache.Builder setValueType(Class valueType) { + /** + * Sets the class type of value. + * @param valueType type of value + * @return builder + */ + public Builder setValueType(Class valueType) { this.valueType = valueType; return this; } - public EhCacheDiskCache.Builder setStoragePath(String storagePath) { + /** + * Desired storage path for disk cache. + * @param storagePath path for disk cache + * @return builder + */ + public Builder setStoragePath(String storagePath) { this.storagePath = storagePath; return this; } - public EhCacheDiskCache.Builder setThreadPoolAlias(String threadPoolAlias) { + /** + * Thread pool alias for the cache. + * @param threadPoolAlias alias + * @return builder + */ + public Builder setThreadPoolAlias(String threadPoolAlias) { this.threadPoolAlias = threadPoolAlias; return this; } - public EhCacheDiskCache.Builder setSettings(Settings settings) { - this.settings = settings; - return this; - } - - public EhCacheDiskCache.Builder setDiskCacheAlias(String diskCacheAlias) { + /** + * Cache alias + * @param diskCacheAlias + * @return builder + */ + public Builder setDiskCacheAlias(String diskCacheAlias) { this.diskCacheAlias = diskCacheAlias; return this; } - public EhCacheDiskCache.Builder setSettingPrefix(String settingPrefix) { - // TODO: Do some basic validation. So that it doesn't end with "." etc. - this.settingPrefix = settingPrefix; - return this; - } - - public EhCacheDiskCache.Builder setIsEventListenerModeSync(boolean isEventListenerModeSync) { + /** + * Determines whether event listener is triggered async/sync. + * @param isEventListenerModeSync mode sync + * @return builder + */ + public Builder setIsEventListenerModeSync(boolean isEventListenerModeSync) { this.isEventListenerModeSync = isEventListenerModeSync; return this; } - public EhCacheDiskCache build() { - return new EhCacheDiskCache<>(this); + @Override + public EhcacheDiskCache build() { + return new EhcacheDiskCache<>(this); } } } diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java new file mode 100644 index 0000000000000..507ecb20f73b9 --- /dev/null +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cache; + +import org.opensearch.common.settings.Setting; + +/** + * Settings related to ehcache. + */ +public class EhcacheSettings { + + static final String SETTING_PREFIX = "cache.disk.ehcache"; + + /** + * Ehcache disk write minimum threads for its pool + */ + public static final Setting DISK_WRITE_MINIMUM_THREADS = Setting.intSetting(SETTING_PREFIX + "min_threads", 2, 1, 5); + + /** + * Ehcache disk write maximum threads for its pool + */ + public static final Setting DISK_WRITE_MAXIMUM_THREADS = Setting.intSetting(SETTING_PREFIX + ".max_threads", 2, 1, 20); + + /** + * Not be to confused with number of disk segments, this is different. Defines + * distinct write queues created for disk store where a group of segments share a write queue. This is + * implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write + * queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool. + * + * Default is 1 within ehcache. + */ + public static final Setting DISK_WRITE_CONCURRENCY = Setting.intSetting(SETTING_PREFIX + ".concurrency", 1, 1, 3); + + /** + * Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but + * will hold that many file pointers. Default is 16. + * + * Default value is 16 within Ehcache. + */ + public static final Setting DISK_SEGMENTS = Setting.intSetting(SETTING_PREFIX + ".segments", 16, 1, 32); + + /** + * Default constructor. Added to fix javadocs. + */ + public EhcacheSettings() {} +} diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/package-info.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/package-info.java new file mode 100644 index 0000000000000..f9be1c3dbf826 --- /dev/null +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Base package for cache plugin */ +package org.opensearch.cache; diff --git a/plugins/cache-ehcache/src/main/plugin-metadata/plugin-security.policy b/plugins/cache-ehcache/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..cbd43446c6ba0 --- /dev/null +++ b/plugins/cache-ehcache/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { + // For ehcache + permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; +}; + + diff --git a/server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhCacheDiskCacheTests.java similarity index 93% rename from server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java rename to plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhCacheDiskCacheTests.java index 12bffb93fc2f8..dc9354e26168a 100644 --- a/server/src/test/java/org/opensearch/common/cache/store/EhCacheDiskCacheTests.java +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhCacheDiskCacheTests.java @@ -6,9 +6,11 @@ * compatible open source license. */ -package org.opensearch.common.cache.store; +package org.opensearch.cache; import org.opensearch.common.cache.LoadAwareCacheLoader; +import org.opensearch.common.cache.store.StoreAwareCache; +import org.opensearch.common.cache.store.StoreAwareCacheRemovalNotification; import org.opensearch.common.cache.store.enums.CacheStoreType; import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; import org.opensearch.common.settings.Settings; @@ -33,18 +35,17 @@ public class EhCacheDiskCacheTests extends OpenSearchSingleNodeTestCase { private static final int CACHE_SIZE_IN_BYTES = 1024 * 101; - private static final String SETTING_PREFIX = "indices.request.cache"; public void testBasicGetAndPut() throws IOException { Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) - .setValueType(String.class) - .setSettings(settings) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") .setThreadPoolAlias("ehcacheTest") .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") - .setSettingPrefix(SETTING_PREFIX) + .setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -79,12 +80,12 @@ public void testConcurrentPut() throws Exception { Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) - .setValueType(String.class) - .setSettings(settings) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") .setThreadPoolAlias("ehcacheTest") .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") - .setSettingPrefix(SETTING_PREFIX) + .setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -122,13 +123,13 @@ public void testEhcacheParallelGets() throws Exception { Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) - .setValueType(String.class) - .setSettings(settings) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") .setThreadPoolAlias("ehcacheTest") .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") - .setSettingPrefix(SETTING_PREFIX) .setIsEventListenerModeSync(true) // For accurate count + .setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -165,12 +166,12 @@ public void testEhcacheParallelGets() throws Exception { public void testEhcacheKeyIterator() throws Exception { Settings settings = Settings.builder().build(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) - .setValueType(String.class) - .setSettings(settings) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") .setThreadPoolAlias("ehcacheTest") - .setSettingPrefix(SETTING_PREFIX) .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(new MockEventListener<>()) @@ -201,13 +202,13 @@ public void testEvictions() throws Exception { Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setIsEventListenerModeSync(true) + .setThreadPoolAlias("ehcacheTest") + .setKeyType(String.class) .setValueType(String.class) .setSettings(settings) - .setThreadPoolAlias("ehcacheTest") - .setSettingPrefix(SETTING_PREFIX) - .setIsEventListenerModeSync(true) - .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -230,13 +231,13 @@ public void testComputeIfAbsentConcurrently() throws Exception { Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) - .setValueType(String.class) - .setSettings(settings) - .setThreadPoolAlias("ehcacheTest") - .setSettingPrefix(SETTING_PREFIX) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") .setIsEventListenerModeSync(true) .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setThreadPoolAlias("ehcacheTest") + .setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -288,7 +289,7 @@ public String load(String key) { } } assertEquals(1, numberOfTimesValueLoaded); - assertEquals(0, ((EhCacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); + assertEquals(0, ((EhcacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); assertEquals(1, mockEventListener.onMissCount.get()); assertEquals(1, mockEventListener.onCachedCount.get()); assertEquals(numberOfRequest - 1, mockEventListener.onHitCount.get()); @@ -300,13 +301,13 @@ public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") + .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setIsEventListenerModeSync(true) + .setThreadPoolAlias("ehcacheTest") + .setKeyType(String.class) .setValueType(String.class) .setSettings(settings) - .setThreadPoolAlias("ehcacheTest") - .setSettingPrefix(SETTING_PREFIX) - .setIsEventListenerModeSync(true) - .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -347,7 +348,7 @@ public String load(String key) throws Exception { phaser.arriveAndAwaitAdvance(); countDownLatch.await(); - assertEquals(0, ((EhCacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); + assertEquals(0, ((EhcacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); ehcacheTest.close(); } } @@ -356,13 +357,13 @@ public void testComputeIfAbsentWithNullValueLoading() throws Exception { Settings settings = Settings.builder().build(); MockEventListener mockEventListener = new MockEventListener<>(); try (NodeEnvironment env = newNodeEnvironment(settings)) { - StoreAwareCache ehcacheTest = new EhCacheDiskCache.Builder().setKeyType(String.class) - .setValueType(String.class) - .setSettings(settings) + StoreAwareCache ehcacheTest = new EhcacheDiskCache.Builder().setDiskCacheAlias("test1") .setThreadPoolAlias("ehcacheTest") - .setSettingPrefix(SETTING_PREFIX) .setIsEventListenerModeSync(true) .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache") + .setKeyType(String.class) + .setValueType(String.class) + .setSettings(settings) .setExpireAfterAccess(TimeValue.MAX_VALUE) .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES) .setEventListener(mockEventListener) @@ -408,7 +409,7 @@ public String load(String key) throws Exception { phaser.arriveAndAwaitAdvance(); countDownLatch.await(); - assertEquals(0, ((EhCacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); + assertEquals(0, ((EhcacheDiskCache) ehcacheTest).getCompletableFutureMap().size()); ehcacheTest.close(); } } @@ -418,21 +419,13 @@ private static String generateRandomString(int length) { StringBuilder randomString = new StringBuilder(length); for (int i = 0; i < length; i++) { - int index = (int) (Math.random() * characters.length()); + int index = (int) (randomDouble() * characters.length()); randomString.append(characters.charAt(index)); } return randomString.toString(); } - // TODO: Remove this from here in final PR. - enum EventType { - ON_HIT, - ON_MISS, - ON_CACHED, - ON_REMOVAL; - } - class MockEventListener implements StoreAwareCacheEventListener { AtomicInteger onMissCount = new AtomicInteger(); diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhcachePluginTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhcachePluginTests.java new file mode 100644 index 0000000000000..61442c3aa66a5 --- /dev/null +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/EhcachePluginTests.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cache; + +import org.opensearch.common.cache.store.StoreAwareCache; +import org.opensearch.common.cache.store.enums.CacheStoreType; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Map; + +public class EhcachePluginTests extends OpenSearchTestCase { + + private EhcacheCachePlugin ehcacheCachePlugin = new EhcacheCachePlugin(); + + public void testGetCacheStoreTypeMap() { + Map factoryMap = ehcacheCachePlugin.getCacheStoreTypeMap(); + assertNotNull(factoryMap); + assertNotNull(factoryMap.get(CacheStoreType.DISK)); + } +} diff --git a/server/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 similarity index 100% rename from server/licenses/slf4j-api-1.7.36.jar.sha1 rename to plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 diff --git a/server/licenses/slf4j-api-LICENSE.txt b/plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt similarity index 100% rename from server/licenses/slf4j-api-LICENSE.txt rename to plugins/discovery-ec2/licenses/slf4j-api-LICENSE.txt diff --git a/server/licenses/slf4j-api-NOTICE.txt b/plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt similarity index 100% rename from server/licenses/slf4j-api-NOTICE.txt rename to plugins/discovery-ec2/licenses/slf4j-api-NOTICE.txt diff --git a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt b/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..2be7689435062 --- /dev/null +++ b/plugins/identity-shiro/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2022 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt b/plugins/identity-shiro/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt b/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..2be7689435062 --- /dev/null +++ b/plugins/ingest-attachment/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2022 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt b/plugins/ingest-attachment/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt b/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..2be7689435062 --- /dev/null +++ b/plugins/repository-azure/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2022 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/repository-azure/licenses/slf4j-api-NOTICE.txt b/plugins/repository-azure/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt b/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..2be7689435062 --- /dev/null +++ b/plugins/repository-hdfs/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2022 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt b/plugins/repository-hdfs/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt b/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..2be7689435062 --- /dev/null +++ b/plugins/repository-s3/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2022 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/repository-s3/licenses/slf4j-api-NOTICE.txt b/plugins/repository-s3/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index 4e503413c265b..8c0ee8ba718ac 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -103,6 +103,13 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', + // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional + 'org.slf4j.helpers.FormattingTuple', + 'org.slf4j.helpers.MessageFormatter', + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + 'org.slf4j.spi.LocationAwareLogger', + 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/plugins/transport-reactor-netty4/build.gradle b/plugins/transport-reactor-netty4/build.gradle index 74ceb2ab5687a..7d7eb330b4a55 100644 --- a/plugins/transport-reactor-netty4/build.gradle +++ b/plugins/transport-reactor-netty4/build.gradle @@ -136,6 +136,13 @@ thirdPartyAudit { 'org.jboss.marshalling.MarshallingConfiguration', 'org.jboss.marshalling.Unmarshaller', + // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional + 'org.slf4j.helpers.FormattingTuple', + 'org.slf4j.helpers.MessageFormatter', + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory', + 'org.slf4j.spi.LocationAwareLogger', + 'com.google.protobuf.nano.CodedOutputByteBufferNano', 'com.google.protobuf.nano.MessageNano', 'com.ning.compress.BufferRecycler', diff --git a/server/build.gradle b/server/build.gradle index 85f16cee83249..e36498bf1038b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -124,9 +124,6 @@ dependencies { api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" - api "org.ehcache:ehcache:${versions.ehcache}" - api "org.slf4j:slf4j-api:${versions.slf4j}" - testImplementation(project(":test:framework")) { // tests use the locally compiled version of server exclude group: 'org.opensearch', module: 'server' diff --git a/server/src/main/java/org/opensearch/common/cache/provider/CacheProvider.java b/server/src/main/java/org/opensearch/common/cache/provider/CacheProvider.java new file mode 100644 index 0000000000000..fd046757a2a53 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/provider/CacheProvider.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.provider; + +import org.opensearch.common.cache.store.StoreAwareCache; +import org.opensearch.common.cache.store.enums.CacheStoreType; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.CachePlugin; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Holds all the cache factories and provides a way to fetch them when needed. + */ +public class CacheProvider { + + private final Map> cacheStoreTypeFactories; + + private final Settings settings; + + public CacheProvider(List cachePlugins, Settings settings) { + this.cacheStoreTypeFactories = getCacheStoreTypeFactories(cachePlugins); + this.settings = settings; + } + + private Map> getCacheStoreTypeFactories(List cachePlugins) { + Map> cacheStoreTypeFactories = new HashMap<>(); + for (CachePlugin cachePlugin : cachePlugins) { + Map factoryMap = cachePlugin.getCacheStoreTypeMap(); + for (Map.Entry entry : factoryMap.entrySet()) { + cacheStoreTypeFactories.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(entry.getValue()); + } + } + return Collections.unmodifiableMap(cacheStoreTypeFactories); + } + + public Map> getCacheStoreTypeFactories() { + return cacheStoreTypeFactories; + } + + /** + * Given a map of storeType and cacheName setting, extract a specific implementation. + * type. + * @param cacheStoreTypeSettings Setting map + * @return CacheStoreType + */ + public Map getCacheStoreType(Map> cacheStoreTypeSettings) { + Map cacheStoreTypeFactoryMap = new HashMap<>(); + for (Map.Entry> cacheStoreTypeFactoryEntry : cacheStoreTypeFactories.entrySet()) { + CacheStoreType cacheStoreType = cacheStoreTypeFactoryEntry.getKey(); + if (!cacheStoreTypeSettings.containsKey(cacheStoreType)) { + continue; + } + for (StoreAwareCache.Factory factory : cacheStoreTypeFactoryEntry.getValue()) { + if (factory.getCacheName().equals(cacheStoreTypeSettings.get(cacheStoreType).get(settings))) { + cacheStoreTypeFactoryMap.put(cacheStoreType, factory); + break; + } + } + } + return cacheStoreTypeFactoryMap; + } + + /** + * Cache types available. + */ + public enum CacheType { + EHCACHE("ehcache"); + + private final String value; + + CacheType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } +} diff --git a/server/src/main/java/org/opensearch/common/cache/provider/package-info.java b/server/src/main/java/org/opensearch/common/cache/provider/package-info.java new file mode 100644 index 0000000000000..24221f222f93f --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/provider/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Base package for cache providers. */ +package org.opensearch.common.cache.provider; diff --git a/server/src/main/java/org/opensearch/common/cache/store/StoreAwareCache.java b/server/src/main/java/org/opensearch/common/cache/store/StoreAwareCache.java index 45ca48d94c140..4cf000dfff8a6 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/StoreAwareCache.java +++ b/server/src/main/java/org/opensearch/common/cache/store/StoreAwareCache.java @@ -8,7 +8,9 @@ package org.opensearch.common.cache.store; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.cache.ICache; +import org.opensearch.common.cache.store.config.StoreAwareCacheConfig; import org.opensearch.common.cache.store.enums.CacheStoreType; /** @@ -18,6 +20,16 @@ * * @opensearch.experimental */ +@ExperimentalApi public interface StoreAwareCache extends ICache { CacheStoreType getTierType(); + + /** + * Provides a way to create a new cache. + */ + interface Factory { + StoreAwareCache create(StoreAwareCacheConfig storeAwareCacheConfig); + + String getCacheName(); + } } diff --git a/server/src/main/java/org/opensearch/common/cache/store/builders/StoreAwareCacheBuilder.java b/server/src/main/java/org/opensearch/common/cache/store/builders/StoreAwareCacheBuilder.java index fc5aa48aae90f..3b52afebf5579 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/builders/StoreAwareCacheBuilder.java +++ b/server/src/main/java/org/opensearch/common/cache/store/builders/StoreAwareCacheBuilder.java @@ -10,6 +10,7 @@ import org.opensearch.common.cache.store.StoreAwareCache; import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; +import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import java.util.function.ToLongBiFunction; @@ -31,6 +32,8 @@ public abstract class StoreAwareCacheBuilder { private StoreAwareCacheEventListener eventListener; + private Settings settings; + public StoreAwareCacheBuilder() {} public StoreAwareCacheBuilder setMaximumWeightInBytes(long sizeInBytes) { @@ -53,6 +56,11 @@ public StoreAwareCacheBuilder setEventListener(StoreAwareCacheEventListene return this; } + public StoreAwareCacheBuilder setSettings(Settings settings) { + this.settings = settings; + return this; + } + public long getMaxWeightInBytes() { return maxWeightInBytes; } @@ -66,7 +74,11 @@ public ToLongBiFunction getWeigher() { } public StoreAwareCacheEventListener getEventListener() { - return eventListener; + return this.eventListener; + } + + public Settings getSettings() { + return settings; } public abstract StoreAwareCache build(); diff --git a/server/src/main/java/org/opensearch/common/cache/store/config/StoreAwareCacheConfig.java b/server/src/main/java/org/opensearch/common/cache/store/config/StoreAwareCacheConfig.java new file mode 100644 index 0000000000000..514c0c1bd8708 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/store/config/StoreAwareCacheConfig.java @@ -0,0 +1,195 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.cache.store.config; + +import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; + +import java.util.function.ToLongBiFunction; + +/** + * Configurations related to store aware caches + * + * @opensearch.internal + */ +public class StoreAwareCacheConfig { + + private long maxWeightInBytes; + + private ToLongBiFunction weigher; + + private TimeValue expireAfterAcess; + + private StoreAwareCacheEventListener eventListener; + + private Settings settings; + + private Class keyType; + + private Class valueType; + + private String storagePath; + + private String threadPoolAlias; + + private String cacheAlias; + + private String settingPrefix; + + // Provides capability to make event listener to run in sync/async mode. + private boolean isEventListenerModeSync; + + private StoreAwareCacheConfig(Builder builder) { + this.cacheAlias = builder.cacheAlias; + this.keyType = builder.keyType; + } + + public long getMaxWeightInBytes() { + return maxWeightInBytes; + } + + public ToLongBiFunction getWeigher() { + return weigher; + } + + public TimeValue getExpireAfterAcess() { + return expireAfterAcess; + } + + public StoreAwareCacheEventListener getEventListener() { + return eventListener; + } + + public boolean isEventListenerModeSync() { + return isEventListenerModeSync; + } + + public Class getKeyType() { + return keyType; + } + + public Class getValueType() { + return valueType; + } + + public String getCacheAlias() { + return cacheAlias; + } + + public Settings getSettings() { + return settings; + } + + public String getSettingPrefix() { + return settingPrefix; + } + + public String getStoragePath() { + return storagePath; + } + + public String getThreadPoolAlias() { + return threadPoolAlias; + } + + /** + * Builder class to build Cache config related parameters. + * @param Type of key. + * @param Type of value. + */ + public static class Builder { + + private long maxWeightInBytes; + + private ToLongBiFunction weigher; + + private TimeValue expireAfterAcess; + + private StoreAwareCacheEventListener eventListener; + + private Settings settings; + + private Class keyType; + + private Class valueType; + + private String storagePath; + + private String threadPoolAlias; + + private String cacheAlias; + + private String settingPrefix; + + // Provides capability to make event listener to run in sync/async mode. + private boolean isEventListenerModeSync; + + public Builder() {} + + public Builder setMaxWeightInBytes(long maxWeightInBytes) { + this.maxWeightInBytes = maxWeightInBytes; + return this; + } + + public Builder setWeigher(ToLongBiFunction weigher) { + this.weigher = weigher; + return this; + } + + public Builder setExpireAfterAcess(TimeValue expireAfterAcess) { + this.expireAfterAcess = expireAfterAcess; + return this; + } + + public Builder setEventListener(StoreAwareCacheEventListener listener) { + this.eventListener = listener; + return this; + } + + public Builder setSettings(Settings settings) { + this.settings = settings; + return this; + } + + public Builder setKeyType(Class keyType) { + this.keyType = keyType; + return this; + } + + public Builder setValueType(Class keyType) { + this.keyType = keyType; + return this; + } + + public Builder setStoragePath(String storagePath) { + this.storagePath = storagePath; + return this; + } + + public Builder setThreadPoolAlias(String threadPoolAlias) { + this.threadPoolAlias = threadPoolAlias; + return this; + } + + public Builder setCacheAlias(String cacheAlias) { + this.cacheAlias = cacheAlias; + return this; + } + + public Builder setIsEventListenerModeSync(boolean isEventListenerModeSync) { + this.isEventListenerModeSync = isEventListenerModeSync; + return this; + } + + public StoreAwareCacheConfig build() { + return new StoreAwareCacheConfig<>(this); + } + } +} diff --git a/server/src/main/java/org/opensearch/common/cache/store/config/package-info.java b/server/src/main/java/org/opensearch/common/cache/store/config/package-info.java new file mode 100644 index 0000000000000..6b662a8af3f9d --- /dev/null +++ b/server/src/main/java/org/opensearch/common/cache/store/config/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Base package for store aware cache config */ +package org.opensearch.common.cache.store.config; diff --git a/server/src/main/java/org/opensearch/common/cache/store/enums/CacheStoreType.java b/server/src/main/java/org/opensearch/common/cache/store/enums/CacheStoreType.java index 04c0825787b66..1e1aeba332c2f 100644 --- a/server/src/main/java/org/opensearch/common/cache/store/enums/CacheStoreType.java +++ b/server/src/main/java/org/opensearch/common/cache/store/enums/CacheStoreType.java @@ -8,11 +8,14 @@ package org.opensearch.common.cache.store.enums; +import org.opensearch.common.annotation.ExperimentalApi; + /** * Cache store types in tiered cache. * * @opensearch.internal */ +@ExperimentalApi public enum CacheStoreType { ON_HEAP, diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 8510122c39fcb..420c0aca3d129 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -83,6 +83,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; import org.opensearch.common.StopWatch; +import org.opensearch.common.cache.provider.CacheProvider; import org.opensearch.common.inject.Injector; import org.opensearch.common.inject.Key; import org.opensearch.common.inject.Module; @@ -178,6 +179,7 @@ import org.opensearch.persistent.PersistentTasksService; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.AnalysisPlugin; +import org.opensearch.plugins.CachePlugin; import org.opensearch.plugins.CircuitBreakerPlugin; import org.opensearch.plugins.ClusterPlugin; import org.opensearch.plugins.CryptoKeyProviderPlugin; @@ -789,6 +791,7 @@ protected Node( final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); + CacheProvider cacheProvider = new CacheProvider(pluginsService.filterPlugins(CachePlugin.class), settings); final IndicesService indicesService = new IndicesService( settings, pluginsService, diff --git a/server/src/main/java/org/opensearch/plugins/CachePlugin.java b/server/src/main/java/org/opensearch/plugins/CachePlugin.java new file mode 100644 index 0000000000000..1774236489048 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/CachePlugin.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.cache.store.StoreAwareCache; +import org.opensearch.common.cache.store.enums.CacheStoreType; + +import java.util.Map; + +/** + * Plugin to extend cache related classes + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface CachePlugin { + + Map getCacheStoreTypeMap(); + + String getName(); +} From c74992f7c3423277b27e6f590bdea02cacf4f8e7 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Thu, 25 Jan 2024 13:25:16 -0800 Subject: [PATCH 16/28] Fixing gradle failures Signed-off-by: Sagar Upadhyaya --- plugins/cache-ehcache/.gitignore | 29 ------------------- .../licenses/slf4j-api-1.7.36.jar.sha1 | 1 + .../crypto-kms/licenses/slf4j-api-LICENSE.txt | 21 ++++++++++++++ .../crypto-kms/licenses/slf4j-api-NOTICE.txt | 0 .../org/opensearch/bootstrap/security.policy | 5 ---- 5 files changed, 22 insertions(+), 34 deletions(-) delete mode 100644 plugins/cache-ehcache/.gitignore create mode 100644 plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt create mode 100644 plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt diff --git a/plugins/cache-ehcache/.gitignore b/plugins/cache-ehcache/.gitignore deleted file mode 100644 index f68d1099657e3..0000000000000 --- a/plugins/cache-ehcache/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -### IntelliJ IDEA ### -out/ -!**/src/main/**/out/ -!**/src/test/**/out/ - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache -bin/ -!**/src/main/**/bin/ -!**/src/test/**/bin/ - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 new file mode 100644 index 0000000000000..77b9917528382 --- /dev/null +++ b/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 @@ -0,0 +1 @@ +6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt b/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt new file mode 100644 index 0000000000000..2be7689435062 --- /dev/null +++ b/plugins/crypto-kms/licenses/slf4j-api-LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) 2004-2022 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt b/plugins/crypto-kms/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index a07ea064928e3..e1226345ef961 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -187,9 +187,4 @@ grant { permission java.io.FilePermission "/sys/fs/cgroup/cpuacct/-", "read"; permission java.io.FilePermission "/sys/fs/cgroup/memory", "read"; permission java.io.FilePermission "/sys/fs/cgroup/memory/-", "read"; - - // For ehcache - permission java.lang.RuntimePermission "createClassLoader"; - permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; - }; From 06386b0304b42cee15571aec163316a111eb0c47 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Thu, 25 Jan 2024 13:32:18 -0800 Subject: [PATCH 17/28] Making ICache extend Closeable Signed-off-by: Sagar Upadhyaya --- .../src/main/java/org/opensearch/common/cache/ICache.java | 6 +++--- .../opensearch/common/cache/tier/TieredSpilloverCache.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/cache/ICache.java b/server/src/main/java/org/opensearch/common/cache/ICache.java index 0eb778034e417..fad2a31786825 100644 --- a/server/src/main/java/org/opensearch/common/cache/ICache.java +++ b/server/src/main/java/org/opensearch/common/cache/ICache.java @@ -10,6 +10,8 @@ import org.opensearch.common.cache.stats.CacheStats; +import java.io.Closeable; + /** * Represents a cache interface. * @param Type of key. @@ -17,7 +19,7 @@ * * @opensearch.experimental */ -public interface ICache { +public interface ICache extends Closeable { V get(K key); void put(K key, V value); @@ -34,7 +36,5 @@ public interface ICache { void refresh(); - void close(); - CacheStats stats(); } diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java index 2216b90b69d69..027eef358c2fa 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredSpilloverCache.java @@ -21,6 +21,7 @@ import org.opensearch.common.util.concurrent.ReleasableLock; import org.opensearch.common.util.iterable.Iterables; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -177,7 +178,7 @@ public void refresh() { } @Override - public void close() { + public void close() throws IOException { for (StoreAwareCache storeAwareCache : cacheList) { storeAwareCache.close(); } From 48185f81a4f448d1aaa11f1e33a2a890fb755e9f Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 26 Jan 2024 14:36:25 -0800 Subject: [PATCH 18/28] Change keystore implementation to live in a module Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 18 ++++++++++++ .../licenses/RoaringBitmap-LICENSE.txt | 0 .../licenses/RoaringBitmap-NOTICE.txt | 0 .../licenses/shims-LICENSE.txt | 0 .../licenses/shims-NOTICE.txt | 0 .../org/opensearch/cache}/KeyStoreStats.java | 3 +- .../cache}/RBMIntKeyLookupStore.java | 29 ++----------------- .../RoaringBitmapKeystoreModulePlugin.java | 13 +++++++++ .../cache}/RBMIntKeyLookupStoreTests.java | 26 ++--------------- server/build.gradle | 4 --- server/licenses/RoaringBitmap-0.9.49.jar.sha1 | 1 - server/licenses/shims-0.9.49.jar.sha1 | 1 - 12 files changed, 38 insertions(+), 57 deletions(-) create mode 100644 modules/roaringbitmap-keystore/build.gradle rename {server => modules/roaringbitmap-keystore}/licenses/RoaringBitmap-LICENSE.txt (100%) rename {server => modules/roaringbitmap-keystore}/licenses/RoaringBitmap-NOTICE.txt (100%) rename {server => modules/roaringbitmap-keystore}/licenses/shims-LICENSE.txt (100%) rename {server => modules/roaringbitmap-keystore}/licenses/shims-NOTICE.txt (100%) rename {server/src/main/java/org/opensearch/common/cache/tier/keystore => modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache}/KeyStoreStats.java (95%) rename {server/src/main/java/org/opensearch/common/cache/tier/keystore => modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache}/RBMIntKeyLookupStore.java (93%) create mode 100644 modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java rename {server/src/test/java/org/opensearch/common/cache/tier/keystore => modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache}/RBMIntKeyLookupStoreTests.java (94%) delete mode 100644 server/licenses/RoaringBitmap-0.9.49.jar.sha1 delete mode 100644 server/licenses/shims-0.9.49.jar.sha1 diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle new file mode 100644 index 0000000000000..b2f72e12a0e23 --- /dev/null +++ b/modules/roaringbitmap-keystore/build.gradle @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +opensearchplugin { + description 'Roaring bitmap-based implementation of a key lookup store, used for tiered caching.' + classname 'org.opensearch.cache.RoaringBitmapKeystorePlugin' + hasClientJar = true +} + +dependencies { + api 'org.roaringbitmap:RoaringBitmap:0.9.49' + runtimeOnly 'org.roaringbitmap:shims:0.9.49' +} diff --git a/server/licenses/RoaringBitmap-LICENSE.txt b/modules/roaringbitmap-keystore/licenses/RoaringBitmap-LICENSE.txt similarity index 100% rename from server/licenses/RoaringBitmap-LICENSE.txt rename to modules/roaringbitmap-keystore/licenses/RoaringBitmap-LICENSE.txt diff --git a/server/licenses/RoaringBitmap-NOTICE.txt b/modules/roaringbitmap-keystore/licenses/RoaringBitmap-NOTICE.txt similarity index 100% rename from server/licenses/RoaringBitmap-NOTICE.txt rename to modules/roaringbitmap-keystore/licenses/RoaringBitmap-NOTICE.txt diff --git a/server/licenses/shims-LICENSE.txt b/modules/roaringbitmap-keystore/licenses/shims-LICENSE.txt similarity index 100% rename from server/licenses/shims-LICENSE.txt rename to modules/roaringbitmap-keystore/licenses/shims-LICENSE.txt diff --git a/server/licenses/shims-NOTICE.txt b/modules/roaringbitmap-keystore/licenses/shims-NOTICE.txt similarity index 100% rename from server/licenses/shims-NOTICE.txt rename to modules/roaringbitmap-keystore/licenses/shims-NOTICE.txt diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java similarity index 95% rename from server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java rename to modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java index 87de1619c2c31..76fbc000245e6 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.common.cache.tier.keystore; +package org.opensearch.cache; import org.opensearch.common.metrics.CounterMetric; @@ -35,3 +35,4 @@ protected KeyStoreStats(long memSizeCapInBytes) { this.numSuccessfulRemovals = new CounterMetric(); } } + diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java similarity index 93% rename from server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java rename to modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java index 63ce12d17ba93..021875539fc18 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java @@ -6,35 +6,11 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +package org.opensearch.cache; -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.cache.tier.keystore; +import org.opensearch.common.cache.tier.keystore.KeyLookupStore; import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Setting; import org.opensearch.core.common.unit.ByteSizeValue; import java.util.HashMap; @@ -407,3 +383,4 @@ protected void setMemSizeCap(ByteSizeValue newMemSizeCap) { } } } + diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java new file mode 100644 index 0000000000000..0cfb255c52ef6 --- /dev/null +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cache; +import org.opensearch.plugins.Plugin; + +public class RoaringBitmapKeystoreModulePlugin extends Plugin { +} diff --git a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java b/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java similarity index 94% rename from server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java rename to modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java index 7626caa14a21f..9c1204f6b9196 100644 --- a/server/src/test/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStoreTests.java +++ b/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java @@ -6,30 +6,7 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.common.cache.tier.keystore; +package org.opensearch.cache; import org.opensearch.common.Randomness; import org.opensearch.common.metrics.CounterMetric; @@ -429,3 +406,4 @@ public void testSetMemSizeCap() throws Exception { assertFalse(kls.add(rand.nextInt())); } } + diff --git a/server/build.gradle b/server/build.gradle index 5fe436ba84f66..e36498bf1038b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -124,10 +124,6 @@ dependencies { api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" - // roaring bitmaps - api 'org.roaringbitmap:RoaringBitmap:0.9.49' - runtimeOnly 'org.roaringbitmap:shims:0.9.49' // might fix complaining about ArraysShims? - testImplementation(project(":test:framework")) { // tests use the locally compiled version of server exclude group: 'org.opensearch', module: 'server' diff --git a/server/licenses/RoaringBitmap-0.9.49.jar.sha1 b/server/licenses/RoaringBitmap-0.9.49.jar.sha1 deleted file mode 100644 index 919a73c074b6a..0000000000000 --- a/server/licenses/RoaringBitmap-0.9.49.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b45b49c1ec5c5fc48580412d0ca635e1833110ea \ No newline at end of file diff --git a/server/licenses/shims-0.9.49.jar.sha1 b/server/licenses/shims-0.9.49.jar.sha1 deleted file mode 100644 index 9e76614ca5207..0000000000000 --- a/server/licenses/shims-0.9.49.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bd7794fbdaa9536354dd2d8d961d9503beb9460 \ No newline at end of file From cdea52dfe7e3cc8825ced318431a8f00c5d2cecf Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 26 Jan 2024 14:40:18 -0800 Subject: [PATCH 19/28] Simplified interface Signed-off-by: Peter Alfonsi --- .../cache/RBMIntKeyLookupStore.java | 4 --- .../cache/tier/keystore/KeyLookupStore.java | 28 ------------------- 2 files changed, 32 deletions(-) diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java index 021875539fc18..f866e27576cc0 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java @@ -167,7 +167,6 @@ public boolean contains(Integer value) { } } - @Override public Integer getInternalRepresentation(Integer value) { if (value == null) { return 0; @@ -252,17 +251,14 @@ public int getSize() { } } - @Override public int getAddAttempts() { return (int) stats.numAddAttempts.count(); } - @Override public int getCollisions() { return (int) stats.numCollisions.count(); } - @Override public boolean isCollision(Integer value1, Integer value2) { if (value1 == null || value2 == null) { return false; diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java index dc2b7a4ba1234..12e0fb37fa4d3 100644 --- a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java +++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java @@ -57,14 +57,6 @@ public interface KeyLookupStore { */ boolean contains(T value); - /** - * Returns the transformed version of the input value, that would be used to stored it in the keystore. - * This transformation should be always be the same for a given instance. - * @param value The value to transform. - * @return The transformed value. - */ - T getInternalRepresentation(T value); - /** * Attempts to safely remove a value from the internal structure, maintaining the property that contains(value) * will never return a false negative. If removing would lead to a false negative, the value won't be removed. @@ -81,26 +73,6 @@ public interface KeyLookupStore { */ int getSize(); - /** - * Returns the number of times add() has been run, including unsuccessful attempts. - * @return The number of adding attempts. - */ - int getAddAttempts(); - - /** - * Returns the number of times add() has returned false due to a collision. - * @return The number of collisions. - */ - int getCollisions(); - - /** - * Checks if two values would collide after being transformed by this store's transformation. - * @param value1 The first value to compare. - * @param value2 The second value to compare. - * @return true if the transformations are equal, false otherwise. - */ - boolean isCollision(T value1, T value2); - /** * Returns an estimate of the store's memory usage. * @return The memory usage From 62b80cad4ea0abc65aff996c9846c85f8be52937 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 26 Jan 2024 15:21:32 -0800 Subject: [PATCH 20/28] SHAs, spotlessApply Signed-off-by: Peter Alfonsi --- .../licenses/RoaringBitmap-0.9.49.jar.sha1 | 1 + .../licenses/shims-0.9.49.jar.sha1 | 1 + .../main/java/org/opensearch/cache/KeyStoreStats.java | 1 - .../org/opensearch/cache/RBMIntKeyLookupStore.java | 6 ++---- .../cache/RoaringBitmapKeystoreModulePlugin.java | 4 ++-- .../opensearch/cache/RBMIntKeyLookupStoreTests.java | 11 ++++++----- 6 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 modules/roaringbitmap-keystore/licenses/RoaringBitmap-0.9.49.jar.sha1 create mode 100644 modules/roaringbitmap-keystore/licenses/shims-0.9.49.jar.sha1 diff --git a/modules/roaringbitmap-keystore/licenses/RoaringBitmap-0.9.49.jar.sha1 b/modules/roaringbitmap-keystore/licenses/RoaringBitmap-0.9.49.jar.sha1 new file mode 100644 index 0000000000000..919a73c074b6a --- /dev/null +++ b/modules/roaringbitmap-keystore/licenses/RoaringBitmap-0.9.49.jar.sha1 @@ -0,0 +1 @@ +b45b49c1ec5c5fc48580412d0ca635e1833110ea \ No newline at end of file diff --git a/modules/roaringbitmap-keystore/licenses/shims-0.9.49.jar.sha1 b/modules/roaringbitmap-keystore/licenses/shims-0.9.49.jar.sha1 new file mode 100644 index 0000000000000..9e76614ca5207 --- /dev/null +++ b/modules/roaringbitmap-keystore/licenses/shims-0.9.49.jar.sha1 @@ -0,0 +1 @@ +8bd7794fbdaa9536354dd2d8d961d9503beb9460 \ No newline at end of file diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java index 76fbc000245e6..bdb7e09751345 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java @@ -35,4 +35,3 @@ protected KeyStoreStats(long memSizeCapInBytes) { this.numSuccessfulRemovals = new CounterMetric(); } } - diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java index f866e27576cc0..a0f5a3b52736a 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java @@ -8,7 +8,6 @@ package org.opensearch.cache; - import org.opensearch.common.cache.tier.keystore.KeyLookupStore; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.core.common.unit.ByteSizeValue; @@ -67,7 +66,6 @@ public int getValue() { // in terms of storage impact static final int REFRESH_SIZE_EST_INTERVAL = 10_000; - // Use this constructor to specify memory cap with default modulo = 2^28, which we found in experiments // to be the best tradeoff between lower memory usage and risk of collisions public RBMIntKeyLookupStore(long memSizeCapInBytes) { @@ -204,7 +202,8 @@ public boolean remove(Integer value) { if (removalSet == null) { // First time a removal has been attempted for this transformed value HashSet newRemovalSet = new HashSet<>(); - newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this transformedValue + newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this + // transformedValue removalSets.put(transformedValue, newRemovalSet); numCollisions.dec(); } else { @@ -379,4 +378,3 @@ protected void setMemSizeCap(ByteSizeValue newMemSizeCap) { } } } - diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java index 0cfb255c52ef6..47c10fd7dded6 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java @@ -7,7 +7,7 @@ */ package org.opensearch.cache; + import org.opensearch.plugins.Plugin; -public class RoaringBitmapKeystoreModulePlugin extends Plugin { -} +public class RoaringBitmapKeystoreModulePlugin extends Plugin {} diff --git a/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java b/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java index 9c1204f6b9196..b5d088ced7041 100644 --- a/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java +++ b/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java @@ -13,7 +13,6 @@ import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.test.OpenSearchTestCase; -import org.roaringbitmap.RoaringBitmap; import java.util.ArrayList; import java.util.HashSet; @@ -22,9 +21,12 @@ import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; +import org.roaringbitmap.RoaringBitmap; + public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase { final int BYTES_IN_MB = 1048576; + public void testInit() { long memCap = 100 * BYTES_IN_MB; RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(memCap); @@ -130,10 +132,10 @@ public void testMemoryCapBlocksAdd() throws Exception { // as that's what our size estimator is designed for. // If we add a run of integers, our size estimator is not valid, especially for small RBMs. - int[] maxEntriesArr = new int[] { 1342000, 100000, 3000000}; + int[] maxEntriesArr = new int[] { 1342000, 100000, 3000000 }; long[] rbmReportedSizes = new long[4]; Random rand = Randomness.get(); - for (int j = 0; j < maxEntriesArr.length; j++) { + for (int j = 0; j < maxEntriesArr.length; j++) { RoaringBitmap rbm = new RoaringBitmap(); for (int i = 0; i < maxEntriesArr[j]; i++) { rbm.add(rand.nextInt()); @@ -144,7 +146,7 @@ public void testMemoryCapBlocksAdd() throws Exception { for (int i = 0; i < maxEntriesArr.length; i++) { double multiplier = RBMIntKeyLookupStore.getRBMSizeMultiplier(maxEntriesArr[i], moduloValue.getValue()); long memSizeCapInBytes = (long) (rbmReportedSizes[i] * multiplier); - //long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue); + // long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue); RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes); for (int j = 0; j < maxEntriesArr[i] + 5000; j++) { kls.add(rand.nextInt()); @@ -406,4 +408,3 @@ public void testSetMemSizeCap() throws Exception { assertFalse(kls.add(rand.nextInt())); } } - From 6be8c6c12e2171fa9bd4f3aea987741d329bf4ac Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 26 Jan 2024 15:25:01 -0800 Subject: [PATCH 21/28] removed commented-out line Signed-off-by: Peter Alfonsi --- .../java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java b/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java index b5d088ced7041..9cece262e518e 100644 --- a/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java +++ b/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java @@ -146,7 +146,6 @@ public void testMemoryCapBlocksAdd() throws Exception { for (int i = 0; i < maxEntriesArr.length; i++) { double multiplier = RBMIntKeyLookupStore.getRBMSizeMultiplier(maxEntriesArr[i], moduloValue.getValue()); long memSizeCapInBytes = (long) (rbmReportedSizes[i] * multiplier); - // long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModuloValue(maxEntries, moduloValue); RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(moduloValue, memSizeCapInBytes); for (int j = 0; j < maxEntriesArr[i] + 5000; j++) { kls.add(rand.nextInt()); From 61cb54dfbbc4ebf6f7603876beb82d56ad224d35 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 26 Jan 2024 15:44:30 -0800 Subject: [PATCH 22/28] javadoc complaints Signed-off-by: Peter Alfonsi --- .../main/java/org/opensearch/cache/KeyStoreStats.java | 7 +++++++ .../org/opensearch/cache/RBMIntKeyLookupStore.java | 11 ++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java index bdb7e09751345..206e0324de4ea 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java @@ -17,12 +17,19 @@ * Getters should be exposed by the KeyLookupStore which uses it. */ public class KeyStoreStats { + // Number of entries protected CounterMetric size; + // Memory cap in bytes protected long memSizeCapInBytes; + // Number of add attempts protected CounterMetric numAddAttempts; + // Number of collisions protected CounterMetric numCollisions; + // True if the store is at capacity protected AtomicBoolean atCapacity; + // Number of removal attempts protected CounterMetric numRemovalAttempts; + // Number of successful removal attempts protected CounterMetric numSuccessfulRemovals; protected KeyStoreStats(long memSizeCapInBytes) { diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java index a0f5a3b52736a..6f453a5dc6c83 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java +++ b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java @@ -49,16 +49,17 @@ public int getValue() { } } + // The modulo applied to values before adding into the RBM protected final int modulo; - protected final int modulo_bitmask; + private final int modulo_bitmask; // Since our modulo is always a power of two we can optimize it by ANDing with a particular bitmask KeyStoreStats stats; - protected RoaringBitmap rbm; + private RoaringBitmap rbm; private HashMap collidedIntCounters; private HashMap> removalSets; - protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - protected final Lock readLock = lock.readLock(); - protected final Lock writeLock = lock.writeLock(); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final Lock readLock = lock.readLock(); + private final Lock writeLock = lock.writeLock(); private long mostRecentByteEstimate; // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms, From ce88ee834223685d2f54498db4f88475630ad42c Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Fri, 26 Jan 2024 16:12:39 -0800 Subject: [PATCH 23/28] fixed name of plugin in build.gradle Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle index b2f72e12a0e23..dd0bb1acf1645 100644 --- a/modules/roaringbitmap-keystore/build.gradle +++ b/modules/roaringbitmap-keystore/build.gradle @@ -8,7 +8,7 @@ opensearchplugin { description 'Roaring bitmap-based implementation of a key lookup store, used for tiered caching.' - classname 'org.opensearch.cache.RoaringBitmapKeystorePlugin' + classname 'org.opensearch.cache.RoaringBitmapKeystoreModulePlugin' hasClientJar = true } From 820169639e8c87522dea1774e231276a20f41997 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 29 Jan 2024 09:21:39 -0800 Subject: [PATCH 24/28] Seeing if applying plugins removes javadoc requirement Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle index dd0bb1acf1645..41408030fe8bd 100644 --- a/modules/roaringbitmap-keystore/build.gradle +++ b/modules/roaringbitmap-keystore/build.gradle @@ -6,6 +6,9 @@ * compatible open source license. */ +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.internal-cluster-test' + opensearchplugin { description 'Roaring bitmap-based implementation of a key lookup store, used for tiered caching.' classname 'org.opensearch.cache.RoaringBitmapKeystoreModulePlugin' @@ -16,3 +19,11 @@ dependencies { api 'org.roaringbitmap:RoaringBitmap:0.9.49' runtimeOnly 'org.roaringbitmap:shims:0.9.49' } + +/*tasks { + javadoc { + options { + (this as CoreJavadocOptions).addStringOption('Xdoclint:none', '-quiet') + } + } +}*/ From 04cd84a579fecfced8ac77309bef533a8fc0195e Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 29 Jan 2024 09:39:40 -0800 Subject: [PATCH 25/28] attempt 2 Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle index 41408030fe8bd..ed63715f67acb 100644 --- a/modules/roaringbitmap-keystore/build.gradle +++ b/modules/roaringbitmap-keystore/build.gradle @@ -6,7 +6,6 @@ * compatible open source license. */ -apply plugin: 'opensearch.yaml-rest-test' apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { From f4e73b9f1c2854c4dcf539bd72faa5d71aa04dd7 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 29 Jan 2024 10:17:25 -0800 Subject: [PATCH 26/28] Attempt 3 Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle index ed63715f67acb..7bee65c50820d 100644 --- a/modules/roaringbitmap-keystore/build.gradle +++ b/modules/roaringbitmap-keystore/build.gradle @@ -6,8 +6,6 @@ * compatible open source license. */ -apply plugin: 'opensearch.internal-cluster-test' - opensearchplugin { description 'Roaring bitmap-based implementation of a key lookup store, used for tiered caching.' classname 'org.opensearch.cache.RoaringBitmapKeystoreModulePlugin' @@ -19,10 +17,10 @@ dependencies { runtimeOnly 'org.roaringbitmap:shims:0.9.49' } -/*tasks { +tasks { javadoc { options { (this as CoreJavadocOptions).addStringOption('Xdoclint:none', '-quiet') } } -}*/ +} From 5261cdf96d99916dca5e9214659156fd108f15fc Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Mon, 29 Jan 2024 12:42:09 -0800 Subject: [PATCH 27/28] attempt 10,000 Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle index 7bee65c50820d..1c569179e492f 100644 --- a/modules/roaringbitmap-keystore/build.gradle +++ b/modules/roaringbitmap-keystore/build.gradle @@ -17,10 +17,9 @@ dependencies { runtimeOnly 'org.roaringbitmap:shims:0.9.49' } -tasks { - javadoc { - options { - (this as CoreJavadocOptions).addStringOption('Xdoclint:none', '-quiet') - } - } +tasks.withType(Javadoc).configureEach { + failOnError false + options.addStringOption('Xdoclint:none', '-quiet') + options.addStringOption('encoding', 'UTF-8') + options.addStringOption('charSet', 'UTF-8') } From c700142004b2b6da2f37c40dd73169dcd4968d88 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Thu, 1 Feb 2024 15:09:11 -0800 Subject: [PATCH 28/28] Moved keystore to the ehcache plugin Signed-off-by: Peter Alfonsi --- modules/roaringbitmap-keystore/build.gradle | 25 ------------------- .../RoaringBitmapKeystoreModulePlugin.java | 13 ---------- plugins/cache-ehcache/build.gradle | 5 ++++ .../licenses/RoaringBitmap-0.9.49.jar.sha1 | 0 .../licenses/RoaringBitmap-LICENSE.txt | 0 .../licenses/RoaringBitmap-NOTICE.txt | 0 .../licenses/shims-0.9.49.jar.sha1 | 0 .../cache-ehcache}/licenses/shims-LICENSE.txt | 0 .../cache-ehcache}/licenses/shims-NOTICE.txt | 0 .../opensearch/cache/EhcacheDiskCache.java | 25 +++++++++++++++---- .../org/opensearch/cache/EhcacheSettings.java | 15 +++++++++++ .../cache/keystore}/KeyStoreStats.java | 2 +- .../cache/keystore}/RBMIntKeyLookupStore.java | 2 +- .../keystore}/RBMIntKeyLookupStoreTests.java | 2 +- 14 files changed, 43 insertions(+), 46 deletions(-) delete mode 100644 modules/roaringbitmap-keystore/build.gradle delete mode 100644 modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java rename {modules/roaringbitmap-keystore => plugins/cache-ehcache}/licenses/RoaringBitmap-0.9.49.jar.sha1 (100%) rename {modules/roaringbitmap-keystore => plugins/cache-ehcache}/licenses/RoaringBitmap-LICENSE.txt (100%) rename {modules/roaringbitmap-keystore => plugins/cache-ehcache}/licenses/RoaringBitmap-NOTICE.txt (100%) rename {modules/roaringbitmap-keystore => plugins/cache-ehcache}/licenses/shims-0.9.49.jar.sha1 (100%) rename {modules/roaringbitmap-keystore => plugins/cache-ehcache}/licenses/shims-LICENSE.txt (100%) rename {modules/roaringbitmap-keystore => plugins/cache-ehcache}/licenses/shims-NOTICE.txt (100%) rename {modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache => plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore}/KeyStoreStats.java (97%) rename {modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache => plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore}/RBMIntKeyLookupStore.java (99%) rename {modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache => plugins/cache-ehcache/src/test/java/org/opensearch/cache/keystore}/RBMIntKeyLookupStoreTests.java (99%) diff --git a/modules/roaringbitmap-keystore/build.gradle b/modules/roaringbitmap-keystore/build.gradle deleted file mode 100644 index 1c569179e492f..0000000000000 --- a/modules/roaringbitmap-keystore/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -opensearchplugin { - description 'Roaring bitmap-based implementation of a key lookup store, used for tiered caching.' - classname 'org.opensearch.cache.RoaringBitmapKeystoreModulePlugin' - hasClientJar = true -} - -dependencies { - api 'org.roaringbitmap:RoaringBitmap:0.9.49' - runtimeOnly 'org.roaringbitmap:shims:0.9.49' -} - -tasks.withType(Javadoc).configureEach { - failOnError false - options.addStringOption('Xdoclint:none', '-quiet') - options.addStringOption('encoding', 'UTF-8') - options.addStringOption('charSet', 'UTF-8') -} diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java b/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java deleted file mode 100644 index 47c10fd7dded6..0000000000000 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RoaringBitmapKeystoreModulePlugin.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.cache; - -import org.opensearch.plugins.Plugin; - -public class RoaringBitmapKeystoreModulePlugin extends Plugin {} diff --git a/plugins/cache-ehcache/build.gradle b/plugins/cache-ehcache/build.gradle index 87c38f616cbb3..562d68e18de7d 100644 --- a/plugins/cache-ehcache/build.gradle +++ b/plugins/cache-ehcache/build.gradle @@ -18,7 +18,12 @@ opensearchplugin { } dependencies { + // for ehcache api "org.ehcache:ehcache:${versions.ehcache}" + + // For roaring bitmaps + api 'org.roaringbitmap:RoaringBitmap:0.9.49' + runtimeOnly 'org.roaringbitmap:shims:0.9.49' } thirdPartyAudit { diff --git a/modules/roaringbitmap-keystore/licenses/RoaringBitmap-0.9.49.jar.sha1 b/plugins/cache-ehcache/licenses/RoaringBitmap-0.9.49.jar.sha1 similarity index 100% rename from modules/roaringbitmap-keystore/licenses/RoaringBitmap-0.9.49.jar.sha1 rename to plugins/cache-ehcache/licenses/RoaringBitmap-0.9.49.jar.sha1 diff --git a/modules/roaringbitmap-keystore/licenses/RoaringBitmap-LICENSE.txt b/plugins/cache-ehcache/licenses/RoaringBitmap-LICENSE.txt similarity index 100% rename from modules/roaringbitmap-keystore/licenses/RoaringBitmap-LICENSE.txt rename to plugins/cache-ehcache/licenses/RoaringBitmap-LICENSE.txt diff --git a/modules/roaringbitmap-keystore/licenses/RoaringBitmap-NOTICE.txt b/plugins/cache-ehcache/licenses/RoaringBitmap-NOTICE.txt similarity index 100% rename from modules/roaringbitmap-keystore/licenses/RoaringBitmap-NOTICE.txt rename to plugins/cache-ehcache/licenses/RoaringBitmap-NOTICE.txt diff --git a/modules/roaringbitmap-keystore/licenses/shims-0.9.49.jar.sha1 b/plugins/cache-ehcache/licenses/shims-0.9.49.jar.sha1 similarity index 100% rename from modules/roaringbitmap-keystore/licenses/shims-0.9.49.jar.sha1 rename to plugins/cache-ehcache/licenses/shims-0.9.49.jar.sha1 diff --git a/modules/roaringbitmap-keystore/licenses/shims-LICENSE.txt b/plugins/cache-ehcache/licenses/shims-LICENSE.txt similarity index 100% rename from modules/roaringbitmap-keystore/licenses/shims-LICENSE.txt rename to plugins/cache-ehcache/licenses/shims-LICENSE.txt diff --git a/modules/roaringbitmap-keystore/licenses/shims-NOTICE.txt b/plugins/cache-ehcache/licenses/shims-NOTICE.txt similarity index 100% rename from modules/roaringbitmap-keystore/licenses/shims-NOTICE.txt rename to plugins/cache-ehcache/licenses/shims-NOTICE.txt diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java index 8ecf38169fe8e..5b43aec7c1ab3 100644 --- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCache.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.cache.keystore.RBMIntKeyLookupStore; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.cache.LoadAwareCacheLoader; @@ -23,6 +24,7 @@ import org.opensearch.common.cache.store.config.StoreAwareCacheConfig; import org.opensearch.common.cache.store.enums.CacheStoreType; import org.opensearch.common.cache.store.listeners.StoreAwareCacheEventListener; +import org.opensearch.common.cache.tier.keystore.KeyLookupStore; import org.opensearch.common.collect.Tuple; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.common.settings.Settings; @@ -61,6 +63,8 @@ import static org.opensearch.cache.EhcacheSettings.DISK_WRITE_CONCURRENCY; import static org.opensearch.cache.EhcacheSettings.DISK_WRITE_MAXIMUM_THREADS; import static org.opensearch.cache.EhcacheSettings.DISK_WRITE_MINIMUM_THREADS; +import static org.opensearch.cache.EhcacheSettings.RBM_KEYSTORE_SIZE; +import static org.opensearch.cache.EhcacheSettings.USE_RBM_KEYSTORE; /** * This variant of disk cache uses Ehcache underneath. @@ -111,6 +115,8 @@ public class EhcacheDiskCache implements StoreAwareCache { */ Map>> completableFutureMap = new ConcurrentHashMap<>(); + KeyLookupStore keystore = null; + private EhcacheDiskCache(Builder builder) { this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null"); this.valueType = Objects.requireNonNull(builder.valueType, "Value type shouldn't be null"); @@ -131,6 +137,10 @@ private EhcacheDiskCache(Builder builder) { this.eventListener = builder.getEventListener(); this.ehCacheEventListener = new EhCacheEventListener(builder.getEventListener()); this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder); + if (USE_RBM_KEYSTORE.get(settings)) { + long keystoreSize = RBM_KEYSTORE_SIZE.get(settings).getBytes(); + this.keystore = new RBMIntKeyLookupStore(keystoreSize); + } } private Cache buildCache(Duration expireAfterAccess, Builder builder) { @@ -216,11 +226,13 @@ public V get(K key) { if (key == null) { throw new IllegalArgumentException("Key passed to ehcache disk cache was null."); } - V value; - try { - value = cache.get(key); - } catch (CacheLoadingException ex) { - throw new OpenSearchException("Exception occurred while trying to fetch item from ehcache disk cache"); + V value = null; + if (keystore == null || keystore.contains(key.hashCode()) || keystore.isFull()) { + try { + value = cache.get(key); + } catch (CacheLoadingException ex) { + throw new OpenSearchException("Exception occurred while trying to fetch item from ehcache disk cache"); + } } if (value != null) { eventListener.onHit(key, value, CacheStoreType.DISK); @@ -239,6 +251,9 @@ public V get(K key) { public void put(K key, V value) { try { cache.put(key, value); + if (keystore != null) { + keystore.add(key.hashCode()); + } } catch (CacheWritingException ex) { throw new OpenSearchException("Exception occurred while put item to ehcache disk cache"); } diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java index 507ecb20f73b9..14c3341def688 100644 --- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheSettings.java @@ -9,6 +9,7 @@ package org.opensearch.cache; import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.unit.ByteSizeValue; /** * Settings related to ehcache. @@ -45,6 +46,20 @@ public class EhcacheSettings { */ public static final Setting DISK_SEGMENTS = Setting.intSetting(SETTING_PREFIX + ".segments", 16, 1, 32); + /** + * Defines whether to use an in-memory keystore to check for probable presence of keys before having to go to disk. + */ + public static final Setting USE_RBM_KEYSTORE = Setting.boolSetting(SETTING_PREFIX + ".use_keystore", true); + + /** + * Defines the max size of the RBM keystore if used (as a percentage of heap memory) + */ + public static final Setting RBM_KEYSTORE_SIZE = Setting.memorySizeSetting( + SETTING_PREFIX + ".keystore_size", + "0.05%", + Setting.Property.Dynamic + ); + /** * Default constructor. Added to fix javadocs. */ diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore/KeyStoreStats.java similarity index 97% rename from modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java rename to plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore/KeyStoreStats.java index 206e0324de4ea..37fdf47163779 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/KeyStoreStats.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore/KeyStoreStats.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.cache; +package org.opensearch.cache.keystore; import org.opensearch.common.metrics.CounterMetric; diff --git a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore/RBMIntKeyLookupStore.java similarity index 99% rename from modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java rename to plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore/RBMIntKeyLookupStore.java index 6f453a5dc6c83..55465d602bcb3 100644 --- a/modules/roaringbitmap-keystore/src/main/java/org/opensearch/cache/RBMIntKeyLookupStore.java +++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/keystore/RBMIntKeyLookupStore.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.cache; +package org.opensearch.cache.keystore; import org.opensearch.common.cache.tier.keystore.KeyLookupStore; import org.opensearch.common.metrics.CounterMetric; diff --git a/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/keystore/RBMIntKeyLookupStoreTests.java similarity index 99% rename from modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java rename to plugins/cache-ehcache/src/test/java/org/opensearch/cache/keystore/RBMIntKeyLookupStoreTests.java index 9cece262e518e..f1a65cfafc3e7 100644 --- a/modules/roaringbitmap-keystore/src/test/java/org/opensearch/cache/RBMIntKeyLookupStoreTests.java +++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/keystore/RBMIntKeyLookupStoreTests.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.cache; +package org.opensearch.cache.keystore; import org.opensearch.common.Randomness; import org.opensearch.common.metrics.CounterMetric;