From 9e982dfddfc95b8bc2f8592078cf65382408942f Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Thu, 28 May 2020 13:45:52 -0400 Subject: [PATCH 1/9] API: Switch to lsxpack_header v206. --- bin/interop-encode.c | 3 ++- lsxpack_header.h | 31 ++++++--------------------- test/test_circ_list.c | 4 ++-- test/test_qpack.c | 50 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 52 insertions(+), 36 deletions(-) diff --git a/bin/interop-encode.c b/bin/interop-encode.c index 9d11744..955a7e3 100644 --- a/bin/interop-encode.c +++ b/bin/interop-encode.c @@ -474,7 +474,8 @@ main (int argc, char **argv) } while (1) { - lsxpack_header_set_ptr(&xhdr, line, tab - line, tab + 1, end - tab - 1); + lsxpack_header_set_offset2(&xhdr, line, 0, tab - line, + tab + 1 - line, end - tab - 1); st = lsqpack_enc_encode(&encoder, enc_buf + enc_off, &enc_sz, hea_buf + hea_off, &hea_sz, &xhdr, 0); switch (st) diff --git a/lsxpack_header.h b/lsxpack_header.h index 020e9ac..8ec7e44 100644 --- a/lsxpack_header.h +++ b/lsxpack_header.h @@ -1,5 +1,5 @@ -#ifndef LSXPACK_HEADER_H_v205 -#define LSXPACK_HEADER_H_v205 +#ifndef LSXPACK_HEADER_H_v206 +#define LSXPACK_HEADER_H_v206 #ifdef __cplusplus extern "C" { @@ -25,7 +25,7 @@ typedef uint32_t lsxpack_strlen_t; enum lsxpack_flag { - LSXPACK_HPACK_IDX = 1, + LSXPACK_HPACK_VAL_MATCHED = 1, LSXPACK_QPACK_IDX = 2, LSXPACK_APP_IDX = 4, LSXPACK_NAME_HASH = 8, @@ -46,7 +46,6 @@ enum lsxpack_flag struct lsxpack_header { char *buf; /* the buffer for headers */ - const char *name_ptr; /* the name pointer can be optionally set for encoding */ uint32_t name_hash; /* hash value for name */ uint32_t nameval_hash; /* hash value for name + value */ lsxpack_strlen_t name_offset; /* the offset for name in the buffer */ @@ -73,7 +72,6 @@ lsxpack_header_set_idx(lsxpack_header_t *hdr, int hpack_idx, hdr->buf = (char *)val; hdr->hpack_index = hpack_idx; assert(hpack_idx != 0); - hdr->flags = LSXPACK_HPACK_IDX; assert(val_len <= LSXPACK_MAX_STRLEN); hdr->val_len = val_len; } @@ -93,21 +91,6 @@ lsxpack_header_set_qpack_idx(lsxpack_header_t *hdr, int qpack_idx, } -static inline void -lsxpack_header_set_ptr(lsxpack_header_t *hdr, - const char *name, size_t name_len, - const char *val, size_t val_len) -{ - memset(hdr, 0, sizeof(*hdr)); - hdr->buf = (char *)val; - assert(val_len <= LSXPACK_MAX_STRLEN); - hdr->val_len = val_len; - hdr->name_ptr = name; - assert(name_len <= LSXPACK_MAX_STRLEN); - hdr->name_len = name_len; -} - - static inline void lsxpack_header_set_offset(lsxpack_header_t *hdr, const char *buf, size_t name_offset, size_t name_len, @@ -160,9 +143,7 @@ lsxpack_header_prepare_decode(lsxpack_header_t *hdr, static inline const char * lsxpack_header_get_name(const lsxpack_header_t *hdr) { - return hdr->name_ptr ? hdr->name_ptr - : (hdr->name_len) ? hdr->buf + hdr->name_offset - : NULL; + return (hdr->name_len)? hdr->buf + hdr->name_offset : NULL; } @@ -178,10 +159,10 @@ static inline void lsxpack_header_mark_val_changed(lsxpack_header_t *hdr) { hdr->flags = (enum lsxpack_flag)(hdr->flags & - ~(LSXPACK_VAL_MATCHED|LSXPACK_NAMEVAL_HASH)); + ~(LSXPACK_HPACK_VAL_MATCHED|LSXPACK_VAL_MATCHED|LSXPACK_NAMEVAL_HASH)); } #ifdef __cplusplus } #endif -#endif //LSXPACK_HEADER_H_v205 +#endif //LSXPACK_HEADER_H_v206 diff --git a/test/test_circ_list.c b/test/test_circ_list.c index 7b5abd3..0e3e9cf 100644 --- a/test/test_circ_list.c +++ b/test/test_circ_list.c @@ -66,9 +66,9 @@ int main(int argc, const char * argv[]) { lsqpack_enc_start_header(&qpackEncoder, 0, i); - name = ":authority"; value = "localhost"; + name = ":authority" "localhost"; - lsxpack_header_set_ptr(&header, name, strlen(name), value, strlen(value)); + lsxpack_header_set_offset2(&header, name, 0, strlen(":authority"), strlen(":authority"), strlen("localhost")); enum lsqpack_enc_status status = lsqpack_enc_encode(&qpackEncoder, &encoderBuffer[usedEncoderSize], &encoderSize, &headerBuffer[usedHeaderSize], &headerSize, &header, (enum lsqpack_enc_flags)0); diff --git a/test/test_qpack.c b/test/test_qpack.c index e813e78..3b08bab 100644 --- a/test/test_qpack.c +++ b/test/test_qpack.c @@ -218,6 +218,31 @@ static const struct qpack_header_block_test }; +struct header_buf +{ + unsigned off; + char buf[UINT16_MAX]; +}; + + +int +header_set_ptr (struct lsxpack_header *hdr, struct header_buf *header_buf, + const char *name, size_t name_len, + const char *val, size_t val_len) +{ + if (header_buf->off + name_len + val_len <= sizeof(header_buf->buf)) + { + memcpy(header_buf->buf + header_buf->off, name, name_len); + memcpy(header_buf->buf + header_buf->off + name_len, val, val_len); + lsxpack_header_set_offset2(hdr, header_buf->buf + header_buf->off, + 0, name_len, name_len, val_len); + header_buf->off += name_len + val_len; + return 0; + } + else + return -1; +} + static void run_header_test (const struct qpack_header_block_test *test) { @@ -233,6 +258,7 @@ run_header_test (const struct qpack_header_block_test *test) float ratio; unsigned char dec_buf[LSQPACK_LONGEST_SDTC]; struct lsxpack_header xhdr; + struct header_buf hbuf; dec_sz = sizeof(dec_buf); s = lsqpack_enc_init(&enc, stderr, test->qhbt_table_size, @@ -251,11 +277,13 @@ run_header_test (const struct qpack_header_block_test *test) header_sz = 0; while (1) { - lsxpack_header_set_ptr(&xhdr, + hbuf.off = 0; + s = header_set_ptr(&xhdr, &hbuf, test->qhbt_headers[i].name, strlen(test->qhbt_headers[i].name), test->qhbt_headers[i].value, strlen(test->qhbt_headers[i].value)); + assert(s == 0); enc_st = lsqpack_enc_encode(&enc, enc_buf + enc_off, &enc_sz, header_buf + header_off, &header_sz, @@ -453,6 +481,7 @@ run_header_cancellation_test(const struct qpack_header_block_test *test) { int s; enum lsqpack_enc_status enc_st; struct lsxpack_header xhdr; + struct header_buf hbuf; s = lsqpack_enc_init(&enc, stderr, 0, 0, test->qhbt_max_risked_streams, LSQPACK_ENC_OPT_IX_AGGR, NULL, NULL); @@ -464,7 +493,8 @@ run_header_cancellation_test(const struct qpack_header_block_test *test) { header_sz = HEADER_BUF_SZ; enc_sz = 0; - lsxpack_header_set_ptr(&xhdr, + hbuf.off = 0; + header_set_ptr(&xhdr, &hbuf, test->qhbt_headers[0].name, strlen(test->qhbt_headers[0].name), test->qhbt_headers[0].value, @@ -553,6 +583,7 @@ test_push_promise (void) size_t header_sz, enc_sz, dec_sz; enum lsqpack_enc_header_flags hflags; struct lsxpack_header xhdr; + struct header_buf hbuf; dec_sz = sizeof(dec_buf); s = lsqpack_enc_init(&enc, stderr, 0x1000, 0x1000, 100, 0, dec_buf, &dec_sz); @@ -564,14 +595,15 @@ test_push_promise (void) assert(0 == s); enc_sz = sizeof(enc_buf); header_sz = sizeof(header_buf); - lsxpack_header_set_ptr(&xhdr, ":method", 7, "dude!", 5); + hbuf.off = 0; + header_set_ptr(&xhdr, &hbuf, ":method", 7, "dude!", 5); enc_st = lsqpack_enc_encode(&enc, enc_buf, &enc_sz, header_buf, &header_sz, &xhdr, 0); assert(LQES_OK == enc_st); enc_sz = sizeof(enc_buf); header_sz = sizeof(header_buf); - lsxpack_header_set_ptr(&xhdr, ":method", 7, "dude!", 5); + header_set_ptr(&xhdr, &hbuf, ":method", 7, "dude!", 5); enc_st = lsqpack_enc_encode(&enc, enc_buf, &enc_sz, header_buf, &header_sz, &xhdr, 0); @@ -586,14 +618,14 @@ test_push_promise (void) assert(0 == s); enc_sz = sizeof(enc_buf); header_sz = sizeof(header_buf); - lsxpack_header_set_ptr(&xhdr, ":method", 7, "dude!", 5); + header_set_ptr(&xhdr, &hbuf, ":method", 7, "dude!", 5); enc_st = lsqpack_enc_encode(&enc, enc_buf, &enc_sz, header_buf, &header_sz, &xhdr, LQEF_NO_HIST_UPD|LQEF_NO_DYN); assert(LQES_OK == enc_st); enc_sz = sizeof(enc_buf); header_sz = sizeof(header_buf); - lsxpack_header_set_ptr(&xhdr, ":method", 7, "where is my car?", 16); + header_set_ptr(&xhdr, &hbuf, ":method", 7, "where is my car?", 16); enc_st = lsqpack_enc_encode(&enc, enc_buf, &enc_sz, header_buf, &header_sz, &xhdr, LQEF_NO_HIST_UPD|LQEF_NO_DYN); @@ -607,7 +639,7 @@ test_push_promise (void) assert(0 == s); enc_sz = sizeof(enc_buf); header_sz = sizeof(header_buf); - lsxpack_header_set_ptr(&xhdr, ":method", 7, "where is my car?", 16); + header_set_ptr(&xhdr, &hbuf, ":method", 7, "where is my car?", 16); enc_st = lsqpack_enc_encode(&enc, enc_buf, &enc_sz, header_buf, &header_sz, &xhdr, 0); @@ -914,6 +946,7 @@ test_enc_risked_streams_test (const char *test) unsigned char *end_cmd; int expect_failure; struct lsxpack_header xhdr; + struct header_buf hbuf; const struct { const char *name; @@ -928,6 +961,7 @@ test_enc_risked_streams_test (const char *test) fprintf(stderr, "BEGIN TEST %s\n", test); lsqpack_enc_preinit(&enc, stderr); + hbuf.off = 0; while (1) { @@ -964,7 +998,7 @@ test_enc_risked_streams_test (const char *test) arg = strtol(test, (char**)&test, 10); sz = sizeof(buf); /* We ignore the output */ - lsxpack_header_set_ptr(&xhdr, + header_set_ptr(&xhdr, &hbuf, headers[arg].name, headers[arg].name_len, headers[arg].value, headers[arg].value_len); es = lsqpack_enc_encode(&enc, buf, &sz, buf, &sz, &xhdr, 0); From a8ae6ef90197df0620ffd51891fb16016b1bb807 Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Thu, 28 May 2020 14:23:09 -0400 Subject: [PATCH 2/9] Update version to 2.2.0 --- lsqpack.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lsqpack.h b/lsqpack.h index 4f72d04..d56baaa 100644 --- a/lsqpack.h +++ b/lsqpack.h @@ -43,8 +43,8 @@ typedef SSIZE_T ssize_t; #endif #define LSQPACK_MAJOR_VERSION 2 -#define LSQPACK_MINOR_VERSION 1 -#define LSQPACK_PATCH_VERSION 2 +#define LSQPACK_MINOR_VERSION 2 +#define LSQPACK_PATCH_VERSION 0 /** Let's start with four billion for now */ typedef unsigned lsqpack_abs_id_t; From 41cec1be498132ab84e70de9ff83b739c6c409bf Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Thu, 15 Oct 2020 08:35:59 -0400 Subject: [PATCH 3/9] Fix a potential divide-by-zero in lsqpack_dec_ratio() --- lsqpack.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lsqpack.c b/lsqpack.c index 9f5152c..850fc36 100644 --- a/lsqpack.c +++ b/lsqpack.c @@ -2888,8 +2888,15 @@ parse_header_data (struct lsqpack_dec *, float lsqpack_dec_ratio (const struct lsqpack_dec *dec) { - if (dec->qpd_bytes_in) - return (float) dec->qpd_bytes_in / (float) dec->qpd_bytes_out; + float ratio; + + if (dec->qpd_bytes_out) + { + ratio = (float) dec->qpd_bytes_in / (float) dec->qpd_bytes_out; + D_DEBUG("bytes in: %u; bytes out: %u, ratio: %.3f", + dec->qpd_bytes_out, dec->qpd_bytes_in, ratio); + return ratio; + } else return 0; } From 93b5499d3a844180c09bee138b35e04ac1d1d6c5 Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Thu, 15 Oct 2020 08:36:33 -0400 Subject: [PATCH 4/9] Fix comment describing struct lsqpack_dec; no code changes --- lsqpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lsqpack.h b/lsqpack.h index d56baaa..c0542ca 100644 --- a/lsqpack.h +++ b/lsqpack.h @@ -765,7 +765,7 @@ struct lsqpack_dec unsigned qpd_cur_capacity; unsigned qpd_max_risked_streams; unsigned qpd_max_entries; - /* Used to calculate estimated compression ratio. Note that the `out' + /* Used to calculate estimated compression ratio. Note that the `in' * part contains bytes sent on the decoder stream, as it also counts * toward the overhead. */ From baf116471eb7d14f1763c75ed9582edc13e5d040 Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Thu, 15 Oct 2020 08:37:23 -0400 Subject: [PATCH 5/9] Fix bytes in/out accounting in encoder and decoder --- lsqpack.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lsqpack.c b/lsqpack.c index 850fc36..395fea6 100644 --- a/lsqpack.c +++ b/lsqpack.c @@ -1272,6 +1272,7 @@ lsqpack_enc_end_header (struct lsqpack_enc *enc, unsigned char *buf, size_t sz, if (qenc_hinfo_at_risk(enc, hinfo)) *header_flags |= LSQECH_REF_AT_RISK; } + enc->qpe_bytes_out += dst - end + sz; return dst - end + sz; } @@ -1290,6 +1291,7 @@ lsqpack_enc_end_header (struct lsqpack_enc *enc, unsigned char *buf, size_t sz, enc->qpe_flags &= ~LSQPACK_ENC_HEADER; if (header_flags) *header_flags = enc->qpe_cur_header.flags; + enc->qpe_bytes_out += 2; return 2; } else @@ -3261,6 +3263,8 @@ header_out_write_value (struct lsqpack_dec *dec, xhdr->flags |= LSXPACK_NAMEVAL_HASH; } r = dec->qpd_dh_if->dhi_process_header(read_ctx->hbrc_hblock, xhdr); + if (r == 0) + dec->qpd_bytes_out += xhdr->name_len + xhdr->val_len; ++read_ctx->hbrc_header_count; memset(&read_ctx->hbrc_out, 0, sizeof(read_ctx->hbrc_out)); if (r != 0) @@ -4120,6 +4124,7 @@ qdec_try_writing_header_ack (struct lsqpack_dec *dec, uint64_t stream_id, if (p > dec_buf) { *dec_buf_sz = p - dec_buf; + dec->qpd_bytes_in += p - dec_buf; return 0; } } @@ -4360,6 +4365,7 @@ lsqpack_dec_write_ici (struct lsqpack_dec *dec, unsigned char *buf, size_t sz) { D_DEBUG("wrote ICI: count=%u", count); dec->qpd_largest_known_id = dec->qpd_last_id; + dec->qpd_bytes_in += p - buf; return p - buf; } else @@ -4418,6 +4424,7 @@ lsqpack_dec_cancel_stream (struct lsqpack_dec *dec, void *hblock, D_DEBUG("cancelled stream %"PRIu64"; generate instruction of %u bytes", read_ctx->hbrc_stream_id, (unsigned) (p - buf)); destroy_header_block_read_ctx(dec, read_ctx); + dec->qpd_bytes_in += p - buf; return p - buf; } else @@ -4450,6 +4457,7 @@ lsqpack_dec_cancel_stream_id (struct lsqpack_dec *dec, uint64_t stream_id, { D_DEBUG("generate Cancel Stream %"PRIu64" instruction of %u bytes", stream_id, (unsigned) (p - buf)); + dec->qpd_bytes_in += p - buf; return p - buf; } else From 76b9a979924cd4df88a564bce0f30141d5050488 Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Thu, 15 Oct 2020 08:42:43 -0400 Subject: [PATCH 6/9] Update version to 2.2.1 --- lsqpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lsqpack.h b/lsqpack.h index c0542ca..415ac9b 100644 --- a/lsqpack.h +++ b/lsqpack.h @@ -44,7 +44,7 @@ typedef SSIZE_T ssize_t; #define LSQPACK_MAJOR_VERSION 2 #define LSQPACK_MINOR_VERSION 2 -#define LSQPACK_PATCH_VERSION 0 +#define LSQPACK_PATCH_VERSION 1 /** Let's start with four billion for now */ typedef unsigned lsqpack_abs_id_t; From 705020b64181e0f38f871af0bfa2d221726c1156 Mon Sep 17 00:00:00 2001 From: Dmitri Tikhonov Date: Fri, 16 Oct 2020 09:01:38 -0400 Subject: [PATCH 7/9] Use the full "LiteSpeed Technologies Inc" in LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 336f2b6..73ce28f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 - 2020 LiteSpeed Tech +Copyright (c) 2018 - 2020 LiteSpeed Technologies Inc Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 1576f779495077f7966bf138ac04b9bdd4206198 Mon Sep 17 00:00:00 2001 From: LiteSpeed Lisa <31540022+lslisa@users.noreply.github.com> Date: Mon, 5 Apr 2021 09:12:53 -0400 Subject: [PATCH 8/9] Added link to new Mnemonic Technique blog post (#39) --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 25c2e47..a58a000 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,7 @@ it before the final RFC is released. ## Functionality ls-qpack is a full-featured, tested, and fast QPACK library. The QPACK encoder -produces excellent compression results based on an innovative mnemonic technique -(to be described in a future article). It boasts the fastest Huffman +produces excellent compression results based on an [innovative mnemonic technique](https://blog.litespeedtech.com/2021/04/05/qpack-mnemonic-technique/). It boasts the fastest Huffman [encoder](https://blog.litespeedtech.com/2019/10/03/fast-huffman-encoder/) and [decoder](https://blog.litespeedtech.com/2019/09/16/fast-huffman-decoder/). From 342be60b9bc86f64e75023b79641da0e807b7a19 Mon Sep 17 00:00:00 2001 From: Aayush Atharva Date: Mon, 5 Apr 2021 18:43:38 +0530 Subject: [PATCH 9/9] Fix typo in README.md (#37) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a58a000..b8c27a0 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Before the QPACK RFC is released, the three parts of the version are: - MINOR: set to the number of QPACK Internet-Draft the lirbary supports; and - PATCH: set to the patch number -Once the RFC is released, MARJO will be set to 1 and the version will follow +Once the RFC is released, MAJOR will be set to 1 and the version will follow the usual MAJOR.MINOR.PATCH pattern. ## API