Merge branch 'master' of github.com:litespeedtech/ls-qpack

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 - 2020 LiteSpeed Tech
+Copyright (c) 2018 - 2020 LiteSpeed Technologies Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -17,8 +17,7 @@ it before the final RFC is released.
 ## Functionality
 
 ls-qpack is a full-featured, tested, and fast QPACK library.  The QPACK encoder
-produces excellent compression results based on an innovative mnemonic technique
-(to be described in a future article).  It boasts the fastest Huffman
+produces excellent compression results based on an [innovative mnemonic technique](https://blog.litespeedtech.com/2021/04/05/qpack-mnemonic-technique/).  It boasts the fastest Huffman
 [encoder](https://blog.litespeedtech.com/2019/10/03/fast-huffman-encoder/) and
 [decoder](https://blog.litespeedtech.com/2019/09/16/fast-huffman-decoder/).
 
@@ -54,7 +53,7 @@ Before the QPACK RFC is released, the three parts of the version are:
 - MINOR: set to the number of QPACK Internet-Draft the lirbary supports; and
 - PATCH: set to the patch number
 
-Once the RFC is released, MARJO will be set to 1 and the version will follow
+Once the RFC is released, MAJOR will be set to 1 and the version will follow
 the usual MAJOR.MINOR.PATCH pattern.
 
 ## API

bin/interop-encode.c

@@ -474,7 +474,8 @@ main (int argc, char **argv)
         }
         while (1)
         {
-            lsxpack_header_set_ptr(&xhdr, line, tab - line, tab + 1, end - tab - 1);
+            lsxpack_header_set_offset2(&xhdr, line, 0, tab - line,
+                                            tab + 1 - line, end - tab - 1);
             st = lsqpack_enc_encode(&encoder, enc_buf + enc_off, &enc_sz,
                         hea_buf + hea_off, &hea_sz, &xhdr, 0);
             switch (st)

lsqpack.c

@@ -1272,6 +1272,7 @@ lsqpack_enc_end_header (struct lsqpack_enc *enc, unsigned char *buf, size_t sz,
             if (qenc_hinfo_at_risk(enc, hinfo))
                 *header_flags |= LSQECH_REF_AT_RISK;
         }
+        enc->qpe_bytes_out += dst - end + sz;
         return dst - end + sz;
     }
 
@@ -1290,6 +1291,7 @@ lsqpack_enc_end_header (struct lsqpack_enc *enc, unsigned char *buf, size_t sz,
         enc->qpe_flags &= ~LSQPACK_ENC_HEADER;
         if (header_flags)
             *header_flags = enc->qpe_cur_header.flags;
+        enc->qpe_bytes_out += 2;
         return 2;
     }
     else
@@ -2888,8 +2890,15 @@ parse_header_data (struct lsqpack_dec *,
 float
 lsqpack_dec_ratio (const struct lsqpack_dec *dec)
 {
-    if (dec->qpd_bytes_in)
-        return (float) dec->qpd_bytes_in / (float) dec->qpd_bytes_out;
+    float ratio;
+
+    if (dec->qpd_bytes_out)
+    {
+        ratio = (float) dec->qpd_bytes_in / (float) dec->qpd_bytes_out;
+        D_DEBUG("bytes in: %u; bytes out: %u, ratio: %.3f",
+                            dec->qpd_bytes_out, dec->qpd_bytes_in, ratio);
+        return ratio;
+    }
     else
         return 0;
 }
@@ -3254,6 +3263,8 @@ header_out_write_value (struct lsqpack_dec *dec,
             xhdr->flags |= LSXPACK_NAMEVAL_HASH;
         }
         r = dec->qpd_dh_if->dhi_process_header(read_ctx->hbrc_hblock, xhdr);
+        if (r == 0)
+            dec->qpd_bytes_out += xhdr->name_len + xhdr->val_len;
         ++read_ctx->hbrc_header_count;
         memset(&read_ctx->hbrc_out, 0, sizeof(read_ctx->hbrc_out));
         if (r != 0)
@@ -4113,6 +4124,7 @@ qdec_try_writing_header_ack (struct lsqpack_dec *dec, uint64_t stream_id,
         if (p > dec_buf)
         {
             *dec_buf_sz = p - dec_buf;
+            dec->qpd_bytes_in += p - dec_buf;
             return 0;
         }
     }
@@ -4353,6 +4365,7 @@ lsqpack_dec_write_ici (struct lsqpack_dec *dec, unsigned char *buf, size_t sz)
         {
             D_DEBUG("wrote ICI: count=%u", count);
             dec->qpd_largest_known_id = dec->qpd_last_id;
+            dec->qpd_bytes_in += p - buf;
             return p - buf;
         }
         else
@@ -4411,6 +4424,7 @@ lsqpack_dec_cancel_stream (struct lsqpack_dec *dec, void *hblock,
         D_DEBUG("cancelled stream %"PRIu64"; generate instruction of %u bytes",
             read_ctx->hbrc_stream_id, (unsigned) (p - buf));
         destroy_header_block_read_ctx(dec, read_ctx);
+        dec->qpd_bytes_in += p - buf;
         return p - buf;
     }
     else
@@ -4443,6 +4457,7 @@ lsqpack_dec_cancel_stream_id (struct lsqpack_dec *dec, uint64_t stream_id,
     {
         D_DEBUG("generate Cancel Stream %"PRIu64" instruction of %u bytes",
             stream_id, (unsigned) (p - buf));
+        dec->qpd_bytes_in += p - buf;
         return p - buf;
     }
     else

lsqpack.h

@@ -43,8 +43,8 @@ typedef SSIZE_T ssize_t;
 #endif
 
 #define LSQPACK_MAJOR_VERSION 2
-#define LSQPACK_MINOR_VERSION 1
-#define LSQPACK_PATCH_VERSION 2
+#define LSQPACK_MINOR_VERSION 2
+#define LSQPACK_PATCH_VERSION 1
 
 /** Let's start with four billion for now */
 typedef unsigned lsqpack_abs_id_t;
@@ -765,7 +765,7 @@ struct lsqpack_dec
     unsigned                qpd_cur_capacity;
     unsigned                qpd_max_risked_streams;
     unsigned                qpd_max_entries;
-    /* Used to calculate estimated compression ratio.  Note that the `out'
+    /* Used to calculate estimated compression ratio.  Note that the `in'
      * part contains bytes sent on the decoder stream, as it also counts
      * toward the overhead.
      */

lsxpack_header.h

@@ -1,5 +1,5 @@
-#ifndef LSXPACK_HEADER_H_v205
-#define LSXPACK_HEADER_H_v205
+#ifndef LSXPACK_HEADER_H_v206
+#define LSXPACK_HEADER_H_v206
 
 #ifdef __cplusplus
 extern "C" {
@@ -25,7 +25,7 @@ typedef uint32_t lsxpack_strlen_t;
 
 enum lsxpack_flag
 {
-    LSXPACK_HPACK_IDX = 1,
+    LSXPACK_HPACK_VAL_MATCHED = 1,
     LSXPACK_QPACK_IDX = 2,
     LSXPACK_APP_IDX   = 4,
     LSXPACK_NAME_HASH = 8,
@@ -46,7 +46,6 @@ enum lsxpack_flag
 struct lsxpack_header
 {
     char             *buf;          /* the buffer for headers */
-    const char       *name_ptr;     /* the name pointer can be optionally set for encoding */
     uint32_t          name_hash;    /* hash value for name */
     uint32_t          nameval_hash; /* hash value for name + value */
     lsxpack_strlen_t  name_offset;  /* the offset for name in the buffer */
@@ -73,7 +72,6 @@ lsxpack_header_set_idx(lsxpack_header_t *hdr, int hpack_idx,
     hdr->buf = (char *)val;
     hdr->hpack_index = hpack_idx;
     assert(hpack_idx != 0);
-    hdr->flags = LSXPACK_HPACK_IDX;
     assert(val_len <= LSXPACK_MAX_STRLEN);
     hdr->val_len = val_len;
 }
@@ -93,21 +91,6 @@ lsxpack_header_set_qpack_idx(lsxpack_header_t *hdr, int qpack_idx,
 }
 
 
-static inline void
-lsxpack_header_set_ptr(lsxpack_header_t *hdr,
-                       const char *name, size_t name_len,
-                       const char *val, size_t val_len)
-{
-    memset(hdr, 0, sizeof(*hdr));
-    hdr->buf = (char *)val;
-    assert(val_len <= LSXPACK_MAX_STRLEN);
-    hdr->val_len = val_len;
-    hdr->name_ptr = name;
-    assert(name_len <= LSXPACK_MAX_STRLEN);
-    hdr->name_len = name_len;
-}
-
-
 static inline void
 lsxpack_header_set_offset(lsxpack_header_t *hdr, const char *buf,
                           size_t name_offset, size_t name_len,
@@ -160,9 +143,7 @@ lsxpack_header_prepare_decode(lsxpack_header_t *hdr,
 static inline const char *
 lsxpack_header_get_name(const lsxpack_header_t *hdr)
 {
-    return hdr->name_ptr ? hdr->name_ptr
-                         : (hdr->name_len) ? hdr->buf + hdr->name_offset
-                                           : NULL;
+    return (hdr->name_len)? hdr->buf + hdr->name_offset : NULL;
 }
 
 
@@ -178,10 +159,10 @@ static inline void
 lsxpack_header_mark_val_changed(lsxpack_header_t *hdr)
 {
     hdr->flags = (enum lsxpack_flag)(hdr->flags &
-                ~(LSXPACK_VAL_MATCHED|LSXPACK_NAMEVAL_HASH));
+       ~(LSXPACK_HPACK_VAL_MATCHED|LSXPACK_VAL_MATCHED|LSXPACK_NAMEVAL_HASH));
 }
 #ifdef __cplusplus
 }
 #endif
 
-#endif //LSXPACK_HEADER_H_v205
+#endif //LSXPACK_HEADER_H_v206

test/test_circ_list.c

@@ -66,9 +66,9 @@ int main(int argc, const char * argv[]) {
 
         lsqpack_enc_start_header(&qpackEncoder, 0, i);
 
-        name = ":authority"; value = "localhost";
+        name = ":authority" "localhost";
 
-        lsxpack_header_set_ptr(&header, name, strlen(name), value, strlen(value));
+        lsxpack_header_set_offset2(&header, name, 0, strlen(":authority"), strlen(":authority"), strlen("localhost"));
 
         enum lsqpack_enc_status status = lsqpack_enc_encode(&qpackEncoder, &encoderBuffer[usedEncoderSize], &encoderSize, &headerBuffer[usedHeaderSize], &headerSize, &header, (enum lsqpack_enc_flags)0);
 

test/test_qpack.c

@@ -218,6 +218,31 @@ static const struct qpack_header_block_test
 };
 
 
+struct header_buf
+{
+    unsigned    off;
+    char        buf[UINT16_MAX];
+};
+
+
+int
+header_set_ptr (struct lsxpack_header *hdr, struct header_buf *header_buf,
+                const char *name, size_t name_len,
+                const char *val, size_t val_len)
+{
+    if (header_buf->off + name_len + val_len <= sizeof(header_buf->buf))
+    {
+        memcpy(header_buf->buf + header_buf->off, name, name_len);
+        memcpy(header_buf->buf + header_buf->off + name_len, val, val_len);
+        lsxpack_header_set_offset2(hdr, header_buf->buf + header_buf->off,
+                                            0, name_len, name_len, val_len);
+        header_buf->off += name_len + val_len;
+        return 0;
+    }
+    else
+        return -1;
+}
+
 static void
 run_header_test (const struct qpack_header_block_test *test)
 {
@@ -233,6 +258,7 @@ run_header_test (const struct qpack_header_block_test *test)
     float ratio;
     unsigned char dec_buf[LSQPACK_LONGEST_SDTC];
     struct lsxpack_header xhdr;
+    struct header_buf hbuf;
 
     dec_sz = sizeof(dec_buf);
     s = lsqpack_enc_init(&enc, stderr, test->qhbt_table_size,
@@ -251,11 +277,13 @@ run_header_test (const struct qpack_header_block_test *test)
         header_sz = 0;
         while (1)
         {
-            lsxpack_header_set_ptr(&xhdr,
+            hbuf.off = 0;
+            s = header_set_ptr(&xhdr, &hbuf,
                     test->qhbt_headers[i].name,
                     strlen(test->qhbt_headers[i].name),
                     test->qhbt_headers[i].value,
                     strlen(test->qhbt_headers[i].value));
+            assert(s == 0);
             enc_st = lsqpack_enc_encode(&enc,
                     enc_buf + enc_off, &enc_sz,
                     header_buf + header_off, &header_sz,
@@ -453,6 +481,7 @@ run_header_cancellation_test(const struct qpack_header_block_test *test) {
     int s;
     enum lsqpack_enc_status enc_st;
     struct lsxpack_header xhdr;
+    struct header_buf hbuf;
 
     s = lsqpack_enc_init(&enc, stderr, 0, 0, test->qhbt_max_risked_streams, 
                          LSQPACK_ENC_OPT_IX_AGGR, NULL, NULL);
@@ -464,7 +493,8 @@ run_header_cancellation_test(const struct qpack_header_block_test *test) {
     header_sz = HEADER_BUF_SZ;
     enc_sz = 0;
 
-    lsxpack_header_set_ptr(&xhdr,
+    hbuf.off = 0;
+    header_set_ptr(&xhdr, &hbuf,
             test->qhbt_headers[0].name,
             strlen(test->qhbt_headers[0].name),
             test->qhbt_headers[0].value,
@@ -553,6 +583,7 @@ test_push_promise (void)
     size_t header_sz, enc_sz, dec_sz;
     enum lsqpack_enc_header_flags hflags;
     struct lsxpack_header xhdr;
+    struct header_buf hbuf;
 
     dec_sz = sizeof(dec_buf);
     s = lsqpack_enc_init(&enc, stderr, 0x1000, 0x1000, 100, 0, dec_buf, &dec_sz);
@@ -564,14 +595,15 @@ test_push_promise (void)
     assert(0 == s);
     enc_sz = sizeof(enc_buf);
     header_sz = sizeof(header_buf);
-    lsxpack_header_set_ptr(&xhdr, ":method", 7, "dude!", 5);
+    hbuf.off = 0;
+    header_set_ptr(&xhdr, &hbuf, ":method", 7, "dude!", 5);
     enc_st = lsqpack_enc_encode(&enc,
             enc_buf, &enc_sz, header_buf, &header_sz,
             &xhdr, 0);
     assert(LQES_OK == enc_st);
     enc_sz = sizeof(enc_buf);
     header_sz = sizeof(header_buf);
-    lsxpack_header_set_ptr(&xhdr, ":method", 7, "dude!", 5);
+    header_set_ptr(&xhdr, &hbuf, ":method", 7, "dude!", 5);
     enc_st = lsqpack_enc_encode(&enc,
             enc_buf, &enc_sz, header_buf, &header_sz,
             &xhdr, 0);
@@ -586,14 +618,14 @@ test_push_promise (void)
     assert(0 == s);
     enc_sz = sizeof(enc_buf);
     header_sz = sizeof(header_buf);
-    lsxpack_header_set_ptr(&xhdr, ":method", 7, "dude!", 5);
+    header_set_ptr(&xhdr, &hbuf, ":method", 7, "dude!", 5);
     enc_st = lsqpack_enc_encode(&enc,
             enc_buf, &enc_sz, header_buf, &header_sz,
             &xhdr, LQEF_NO_HIST_UPD|LQEF_NO_DYN);
     assert(LQES_OK == enc_st);
     enc_sz = sizeof(enc_buf);
     header_sz = sizeof(header_buf);
-    lsxpack_header_set_ptr(&xhdr, ":method", 7, "where is my car?", 16);
+    header_set_ptr(&xhdr, &hbuf, ":method", 7, "where is my car?", 16);
     enc_st = lsqpack_enc_encode(&enc,
             enc_buf, &enc_sz, header_buf, &header_sz,
             &xhdr, LQEF_NO_HIST_UPD|LQEF_NO_DYN);
@@ -607,7 +639,7 @@ test_push_promise (void)
     assert(0 == s);
     enc_sz = sizeof(enc_buf);
     header_sz = sizeof(header_buf);
-    lsxpack_header_set_ptr(&xhdr, ":method", 7, "where is my car?", 16);
+    header_set_ptr(&xhdr, &hbuf, ":method", 7, "where is my car?", 16);
     enc_st = lsqpack_enc_encode(&enc,
             enc_buf, &enc_sz, header_buf, &header_sz,
             &xhdr, 0);
@@ -914,6 +946,7 @@ test_enc_risked_streams_test (const char *test)
     unsigned char *end_cmd;
     int expect_failure;
     struct lsxpack_header xhdr;
+    struct header_buf hbuf;
 
     const struct {
         const char *name;
@@ -928,6 +961,7 @@ test_enc_risked_streams_test (const char *test)
 
     fprintf(stderr, "BEGIN TEST %s\n", test);
     lsqpack_enc_preinit(&enc, stderr);
+    hbuf.off = 0;
 
     while (1)
     {
@@ -964,7 +998,7 @@ test_enc_risked_streams_test (const char *test)
             arg = strtol(test, (char**)&test, 10);
             sz = sizeof(buf);
             /* We ignore the output */
-            lsxpack_header_set_ptr(&xhdr,
+            header_set_ptr(&xhdr, &hbuf,
                         headers[arg].name, headers[arg].name_len,
                         headers[arg].value, headers[arg].value_len);
             es = lsqpack_enc_encode(&enc, buf, &sz, buf, &sz, &xhdr, 0);