Require compressed forms to be encoded canonically when deserializing

This should prevent potential grinding attacks where some non-canonical
encodings of a compressed form could be used to change its hash and thus
the next challenges derived from it.

Canonically encoded compressed forms must be reduced and must produce
the same string when deserialized and serialized again.

src/bqfc.c

@@ -240,6 +240,17 @@ int bqfc_serialize(uint8_t *out_str, mpz_t a, mpz_t b, size_t d_bits)
     return ret;
 }
 
+static int bqfc_verify_canon(mpz_t a, mpz_t b, const uint8_t *str, size_t d_bits)
+{
+    uint8_t canon_str[BQFC_FORM_SIZE];
+    int ret = bqfc_serialize(canon_str, a, b, d_bits);
+
+    if (ret)
+        return ret;
+
+    return memcmp(canon_str, str, BQFC_FORM_SIZE);
+}
+
 int bqfc_deserialize(mpz_t out_a, mpz_t out_b, const mpz_t D, const uint8_t *str, size_t size, size_t d_bits)
 {
     struct qfb_c f_c;
@@ -262,6 +273,10 @@ int bqfc_deserialize(mpz_t out_a, mpz_t out_b, const mpz_t D, const uint8_t *str
         goto out;
 
     ret = bqfc_decompr(out_a, out_b, D, &f_c);
+    if (ret)
+        goto out;
+
+    ret = bqfc_verify_canon(out_a, out_b, str, d_bits);
 out:
     mpz_clears(f_c.a, f_c.t, f_c.g, f_c.b0, NULL);
     return ret;

src/proof_common.h

@@ -56,7 +56,11 @@ form DeserializeForm(const integer &D, const uint8_t *bytes, size_t size)
     if (bqfc_deserialize(a.impl, b.impl, D.impl, bytes, size, D.num_bits())) {
         throw std::runtime_error("Deserializing compressed form failed");
     }
-    return form::from_abd(a, b, D);
+    form f = form::from_abd(a, b, D);
+    if (!f.is_reduced()) {
+        throw std::runtime_error("Form is not reduced");
+    }
+    return f;
 }
 
 integer FastPow(uint64_t a, uint64_t b, integer& c) {

src/vdf_new.h

@@ -136,6 +136,18 @@ struct form {
         ::reduce(a, b, c);
     }
 
+    bool is_reduced() {
+        int a_cmp_c = mpz_cmp(a.impl, c.impl);
+
+        // a <= c, and if a == c, then b >= 0
+        if (a_cmp_c < 0 || (a_cmp_c == 0 && mpz_sgn(b.impl) >= 0)) {
+            // -a < b <= a
+            if (mpz_cmpabs(a.impl, b.impl) > 0 || mpz_cmp(a.impl, b.impl) == 0)
+                return true;
+        }
+        return false;
+    }
+
     form inverse() const {
         form res=*this;
         res.b=-res.b;