Skip to content

Commit

Permalink
Merge branch 'unstable' into hnsw-query
Browse files Browse the repository at this point in the history
  • Loading branch information
PragmaTwice authored Aug 1, 2024
2 parents 77d0f65 + a3863f9 commit 17e1fdd
Show file tree
Hide file tree
Showing 13 changed files with 1,197 additions and 11 deletions.
30 changes: 30 additions & 0 deletions .github/workflows/kvrocks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,12 @@ jobs:
- name: ArchLinux
image: archlinux:base
compiler: gcc
- name: Rocky Linux 8
image: rockylinux:8
compiler: gcc
- name: Rocky Linux 9
image: rockylinux:9
compiler: gcc

runs-on: ubuntu-22.04
container:
Expand All @@ -448,6 +454,30 @@ jobs:
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup Rocky Linux 8
if: ${{ startsWith(matrix.image, 'rockylinux:8') }}
run: |
dnf install -y epel-release
dnf config-manager --set-enabled powertools
dnf install -y git gcc-toolset-12 autoconf automake libtool libstdc++-static python3 python3-pip openssl-devel which cmake
source /opt/rh/gcc-toolset-12/enable
update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
update-alternatives --install /usr/bin/cc cc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100
update-alternatives --install /usr/bin/c++ c++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Setup Rocky Linux 9
if: ${{ startsWith(matrix.image, 'rockylinux:9') }}
run: |
dnf install -y epel-release
dnf config-manager --set-enabled crb
dnf install -y git gcc-toolset-12 autoconf automake libtool libstdc++-static python3 python3-pip openssl-devel which cmake
source /opt/rh/gcc-toolset-12/enable
update-alternatives --install /usr/bin/g++ g++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
update-alternatives --install /usr/bin/cc cc /opt/rh/gcc-toolset-12/root/usr/bin/gcc 100
update-alternatives --install /usr/bin/c++ c++ /opt/rh/gcc-toolset-12/root/usr/bin/g++ 100
echo "NPROC=$(nproc)" >> $GITHUB_ENV
- name: Cache redis
id: cache-redis
uses: actions/cache@v3
Expand Down
14 changes: 7 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,20 +42,20 @@ endif()
find_package(Backtrace REQUIRED)

if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
message(FATAL_ERROR "It is expected to build kvrocks with GCC 7 or above")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
message(FATAL_ERROR "It is expected to build kvrocks with GCC 8 or above")
endif()
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)
message(FATAL_ERROR "It is expected to build kvrocks with Clang 5 or above")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
message(FATAL_ERROR "It is expected to build kvrocks with Clang 8 or above")
endif()
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10)
message(FATAL_ERROR "It is expected to build kvrocks with Xcode toolchains 10 or above")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
message(FATAL_ERROR "It is expected to build kvrocks with Xcode toolchains 11 or above")
endif()
else()
message(WARNING "The compiler you are currently using is not officially supported,
so you can try switching to GCC>=7 or Clang>=5 if you encounter problems")
so you can try switching to GCC>=8 or Clang>=8 if you encounter problems")
endif()

if(CMAKE_GENERATOR STREQUAL "Ninja")
Expand Down
103 changes: 103 additions & 0 deletions src/commands/cmd_hll.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/

#include <types/redis_hyperloglog.h>

#include <algorithm>

#include "commander.h"
#include "commands/command_parser.h"
#include "server/redis_reply.h"
#include "server/server.h"

namespace redis {

/// PFADD key [element [element ...]]
/// Complexity: O(1) for each element added.
class CommandPfAdd final : public Commander {
public:
Status Execute(Server *srv, Connection *conn, std::string *output) override {
redis::HyperLogLog hll(srv->storage, conn->GetNamespace());
std::vector<uint64_t> hashes(args_.size() - 1);
for (size_t i = 1; i < args_.size(); i++) {
hashes[i - 1] = redis::HyperLogLog::HllHash(args_[i]);
}
uint64_t ret{};
auto s = hll.Add(args_[0], hashes, &ret);
if (!s.ok() && !s.IsNotFound()) {
return {Status::RedisExecErr, s.ToString()};
}
*output = redis::Integer(ret);
return Status::OK();
}
};

/// PFCOUNT key [key ...]
/// Complexity: O(1) with a very small average constant time when called with a single key.
/// O(N) with N being the number of keys, and much bigger constant times,
/// when called with multiple keys.
class CommandPfCount final : public Commander {
Status Execute(Server *srv, Connection *conn, std::string *output) override {
redis::HyperLogLog hll(srv->storage, conn->GetNamespace());
uint64_t ret{};
rocksdb::Status s;
if (args_.size() > 1) {
std::vector<Slice> keys(args_.begin(), args_.end());
s = hll.CountMultiple(keys, &ret);
} else {
s = hll.Count(args_[0], &ret);
}
if (!s.ok() && !s.IsNotFound()) {
return {Status::RedisExecErr, s.ToString()};
}
if (s.IsNotFound()) {
ret = 0;
}
*output = redis::Integer(ret);
return Status::OK();
}
};

/// PFMERGE destkey [sourcekey [sourcekey ...]]
///
/// complexity: O(N) to merge N HyperLogLogs, but with high constant times.
class CommandPfMerge final : public Commander {
Status Execute(Server *srv, Connection *conn, std::string *output) override {
redis::HyperLogLog hll(srv->storage, conn->GetNamespace());
std::vector<std::string> keys(args_.begin() + 1, args_.end());
std::vector<Slice> src_user_keys;
src_user_keys.reserve(args_.size() - 1);
for (size_t i = 1; i < args_.size(); i++) {
src_user_keys.emplace_back(args_[i]);
}
auto s = hll.Merge(/*dest_user_key=*/args_[0], src_user_keys);
if (!s.ok() && !s.IsNotFound()) {
return {Status::RedisExecErr, s.ToString()};
}
*output = redis::SimpleString("OK");
return Status::OK();
}
};

REDIS_REGISTER_COMMANDS(MakeCmdAttr<CommandPfAdd>("pfadd", -2, "write", 1, 1, 1),
MakeCmdAttr<CommandPfCount>("pfcount", -2, "read-only", 1, -1, 1),
MakeCmdAttr<CommandPfMerge>("pfmerge", -2, "write", 1, -1, 1), );

} // namespace redis
25 changes: 24 additions & 1 deletion src/storage/redis_metadata.cc
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ bool Metadata::ExpireAt(uint64_t expired_ts) const {
bool Metadata::IsSingleKVType() const { return Type() == kRedisString || Type() == kRedisJson; }

bool Metadata::IsEmptyableType() const {
return IsSingleKVType() || Type() == kRedisStream || Type() == kRedisBloomFilter;
return IsSingleKVType() || Type() == kRedisStream || Type() == kRedisBloomFilter || Type() == kRedisHyperLogLog;
}

bool Metadata::Expired() const { return ExpireAt(util::GetTimeStampMS()); }
Expand Down Expand Up @@ -472,3 +472,26 @@ rocksdb::Status JsonMetadata::Decode(Slice *input) {

return rocksdb::Status::OK();
}

void HyperLogLogMetadata::Encode(std::string *dst) const {
Metadata::Encode(dst);
PutFixed8(dst, static_cast<uint8_t>(this->encode_type));
}

rocksdb::Status HyperLogLogMetadata::Decode(Slice *input) {
if (auto s = Metadata::Decode(input); !s.ok()) {
return s;
}

uint8_t encoded_type = 0;
if (!GetFixed8(input, &encoded_type)) {
return rocksdb::Status::InvalidArgument(kErrMetadataTooShort);
}
// Check validity of encode type
if (encoded_type > 0) {
return rocksdb::Status::InvalidArgument(fmt::format("Invalid encode type {}", encoded_type));
}
this->encode_type = static_cast<EncodeType>(encoded_type);

return rocksdb::Status::OK();
}
26 changes: 24 additions & 2 deletions src/storage/redis_metadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ enum RedisType : uint8_t {
kRedisStream = 8,
kRedisBloomFilter = 9,
kRedisJson = 10,
kRedisHyperLogLog = 11,
};

struct RedisTypes {
Expand Down Expand Up @@ -90,8 +91,9 @@ enum RedisCommand {
kRedisCmdLMove,
};

const std::vector<std::string> RedisTypeNames = {"none", "string", "hash", "list", "set", "zset",
"bitmap", "sortedint", "stream", "MBbloom--", "ReJSON-RL"};
const std::vector<std::string> RedisTypeNames = {"none", "string", "hash", "list",
"set", "zset", "bitmap", "sortedint",
"stream", "MBbloom--", "ReJSON-RL", "hyperloglog"};

constexpr const char *kErrMsgWrongType = "WRONGTYPE Operation against a key holding the wrong kind of value";
constexpr const char *kErrMsgKeyExpired = "the key was expired";
Expand Down Expand Up @@ -313,3 +315,23 @@ class JsonMetadata : public Metadata {
void Encode(std::string *dst) const override;
rocksdb::Status Decode(Slice *input) override;
};

class HyperLogLogMetadata : public Metadata {
public:
enum class EncodeType : uint8_t {
// Redis-style dense encoding implement as bitmap like sub keys to
// store registers by segment in data column family.
// The registers are stored in 6-bit format and each segment contains
// 768 registers.
DENSE = 0,
// TODO(mwish): sparse encoding
// SPARSE = 1,
};

explicit HyperLogLogMetadata(bool generate_version = true) : Metadata(kRedisHyperLogLog, generate_version) {}

void Encode(std::string *dst) const override;
rocksdb::Status Decode(Slice *input) override;

EncodeType encode_type = EncodeType::DENSE;
};
4 changes: 4 additions & 0 deletions src/storage/storage.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@
#include "observer_or_unique.h"
#include "status.h"

#if defined(__sparc__) || defined(__arm__)
#define USE_ALIGNED_ACCESS
#endif

enum class StorageEngineType : uint16_t {
RocksDB,
Speedb,
Expand Down
Loading

0 comments on commit 17e1fdd

Please sign in to comment.