chore: update to icx-proxy-dev built along with rest of replica binar…

…ies (#3246) This updates to the dev version of the icx-proxy, at a commit matching the replica and other released binaries. It also updates the replica update script to update the icx-proxy in the future. The new icx-proxy checks certificates. This has exposed some problems in the asset canister, tracked here and described in the changelog: - https://dfinity.atlassian.net/browse/SDK-1240 - https://dfinity.atlassian.net/browse/SDK-1245 - https://dfinity.atlassian.net/browse/SDK-1246 So far these appear to be edge cases. Since they match the behavior that will be seen on mainnet anyway, it seems better to see this behavior in local development as well. Fixes https://dfinity.atlassian.net/browse/SDK-1200
dfinity · Sep 28, 2023 · 6024938 · 6024938
1 parent d2404ec
commit 6024938
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 31 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -67,6 +67,18 @@ Removed this warning: "Project-specific networks are deprecated and will be remo
 
 ## Dependencies
 
+### icx-proxy
+
+Updated to a version of the icx-proxy that is released with the replica and other related binaries.
+
+Changes in behavior:
+- "%%" is no longer accepted when url-decoding filenames for the asset canister.  Though curl supports this, it's not part of the standard. Please replace with %25.
+- The icx-proxy now performs response verification.  This has exposed some bugs in the asset canister.  However, since this new icx-proxy matches what the boundary nodes use, this will better match the behavior seen on the mainnet.
+- Bugs that this has exposed in the asset canister:
+  - after disabling aliasing for an asset, the asset canister will return an incorrect certification in the 404 response.
+  - after setting a custom "etag" header in .ic-assets.json, the asset canister will return an incorrect certification in the 200 response.
+  - assets with certain characters in the filename (example: "æ") will no longer be served correctly.  The definition of "certain characters" is not yet known.
+
 ### Frontend canister
 
 For certification v1, if none of the requested encoding are certified but another encoding is certified, then the frontend canister once again returns the certificatie even though the response hash won't match.

diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash
@@ -694,6 +694,36 @@ check_permission_failure() {
   assert_command dfx deploy
 }
 
+@test "can serve filenames with special characters in filename" {
+  # This is observed, not expected behavior
+  # see https://dfinity.atlassian.net/browse/SDK-1247
+  install_asset assetscanister
+
+  dfx_start
+
+  echo "filename is an ae symbol" >'src/e2e_project_frontend/assets/æ'
+
+  dfx deploy
+
+  dfx canister  call --query e2e_project_frontend list '(record {})'
+
+  # decode as expected
+  assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/%e6";headers=vec{};method="GET";body=vec{}})'
+  assert_match "filename is an ae symbol" # candid looks like blob "filename is \c3\a6\0a"
+
+  ID=$(dfx canister id e2e_project_frontend)
+  PORT=$(get_webserver_port)
+
+  # fails with Err(InvalidExpressionPath)
+  assert_command_fail curl --fail -vv http://localhost:"$PORT"/%c3%a6?canisterId="$ID"
+
+  # fails with Err(Utf8ConversionError(FromUtf8Error { bytes: [47, 230], error: Utf8Error { valid_up_to: 1, error_len: None } }))
+  assert_command_fail curl --fail -vv http://localhost:"$PORT"/%e6?canisterId="$ID"
+  # assert_match "200 OK" "$stderr"
+  # assert_match "filename is an ae symbol"
+  assert_contains "500 Internal Server Error"
+}
+
 @test "http_request percent-decodes urls" {
   install_asset assetscanister
 
@@ -717,11 +747,11 @@ check_permission_failure() {
   assert_match "contents of file with plus in filename"
   assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/has%2Bplus.txt";headers=vec{};method="GET";body=vec{}})'
   assert_match "contents of file with plus in filename"
-  assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/has%%percent.txt";headers=vec{};method="GET";body=vec{}})'
+  assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/has%25percent.txt";headers=vec{};method="GET";body=vec{}})'
   assert_match "contents of file with percent in filename"
   assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/%e6";headers=vec{};method="GET";body=vec{}})'
   assert_match "filename is an ae symbol" # candid looks like blob "filename is \c3\a6\0a"
-  assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/%%";headers=vec{};method="GET";body=vec{}})'
+  assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/%25";headers=vec{};method="GET";body=vec{}})'
   assert_match "filename is percent"
    # this test ensures url decoding happens after removing the query string
   assert_command dfx canister call --query e2e_project_frontend http_request '(record{url="/filename%3fwithqmark.txt";headers=vec{};method="GET";body=vec{}})'
@@ -755,15 +785,18 @@ check_permission_failure() {
   assert_match "200 OK" "$stderr"
   assert_match "contents of file with plus in filename"
 
-  assert_command curl --fail -vv http://localhost:"$PORT"/has%%percent.txt?canisterId="$ID"
+  assert_command curl --fail -vv http://localhost:"$PORT"/has%25percent.txt?canisterId="$ID"
   assert_match "200 OK" "$stderr"
   assert_match "contents of file with percent in filename"
 
-  assert_command curl --fail -vv http://localhost:"$PORT"/%e6?canisterId="$ID"
-  assert_match "200 OK" "$stderr"
-  assert_match "filename is an ae symbol"
+  assert_command_fail curl --fail -vv http://localhost:"$PORT"/%e6?canisterId="$ID"
+  # see https://dfinity.atlassian.net/browse/SDK-1247
+  # fails with Err(Utf8ConversionError(FromUtf8Error { bytes: [47, 230], error: Utf8Error { valid_up_to: 1, error_len: None } }))
+  assert_contains "500 Internal Server Error"
+  # assert_match "200 OK" "$stderr"
+  # assert_match "filename is an ae symbol"
 
-  assert_command curl --fail -vv http://localhost:"$PORT"/%%?canisterId="$ID"
+  assert_command curl --fail -vv http://localhost:"$PORT"/%25?canisterId="$ID"
   assert_match "200 OK" "$stderr"
   assert_match "filename is percent symbol"
 
@@ -1193,24 +1226,59 @@ CHERRIES" "$stdout"
   assert_match "404 Not Found"
   assert_command curl -vv "http://localhost:$PORT/.well-known/.another-hidden/ignored.txt?canisterId=$ID"
   assert_match "404 Not Found"
+}
+
+@test "asset configuration via .ic-assets.json5 - overwriting etag breaks certification" {
+  # this is observed behavior, not expected behavior
+  # https://dfinity.atlassian.net/browse/SDK-1245
+  install_asset assetscanister
+
+  dfx_start
+
+  touch src/e2e_project_frontend/assets/thing.json
+
+  dfx deploy
+
+  ID=$(dfx canister id e2e_project_frontend)
+  PORT=$(get_webserver_port)
+
+  dfx canister  call --query e2e_project_frontend http_request '(record{url="/thing.json";headers=vec{};method="GET";body=vec{}})'
+  assert_command curl --fail --head "http://localhost:$PORT/thing.json?canisterId=$ID"
+
+  echo '[
+    {
+      "match": "thing.json",
+      "headers": {
+        "etag": "my-etag"
+      }
+    }
+  ]' > src/e2e_project_frontend/assets/.ic-assets.json5
+
+  dfx deploy
+
+  dfx canister call --query e2e_project_frontend http_request '(record{url="/thing.json";headers=vec{};method="GET";body=vec{}})'
 
+  assert_command_fail curl --fail --head "http://localhost:$PORT/thing.json?canisterId=$ID"
+  assert_contains "500 Internal Server Error"
 }
+
 @test "asset configuration via .ic-assets.json5 - overwriting default headers" {
   install_asset assetscanister
 
   dfx_start
 
   touch src/e2e_project_frontend/assets/thing.json
 
+  # this test used to also set etag, but that breaks certification
+  # see https://dfinity.atlassian.net/browse/SDK-1245
   echo '[
     {
       "match": "thing.json",
       "cache": { "max_age": 2000 },
       "headers": {
         "Content-Encoding": "my-encoding",
         "Content-Type": "x-type",
-        "Cache-Control": "custom",
-        "etag": "my-etag"
+        "Cache-Control": "custom"
       }
     }
   ]' > src/e2e_project_frontend/assets/.ic-assets.json5
@@ -1220,11 +1288,13 @@ CHERRIES" "$stdout"
   ID=$(dfx canister id e2e_project_frontend)
   PORT=$(get_webserver_port)
 
-  assert_command curl --head "http://localhost:$PORT/thing.json?canisterId=$ID"
+  dfx canister call --query e2e_project_frontend http_request '(record{url="/thing.json";headers=vec{};method="GET";body=vec{}})'
+
+  assert_command curl --fail --head "http://localhost:$PORT/thing.json?canisterId=$ID"
   assert_match "cache-control: custom"
   assert_match "content-encoding: my-encoding"
   assert_match "content-type: x-type"
-  assert_not_match "etag: my-etag"
+  # https://dfinity.atlassian.net/browse/SDK-1245 assert_not_match "etag: my-etag"
   assert_match "etag: \"[a-z0-9]{64}\""
 }
 
@@ -1271,9 +1341,58 @@ CHERRIES" "$stdout"
   assert_match "test index file"
 
   # toggle aliasing on and off using `set_asset_properties`
+  # this doesn't work, see https://dfinity.atlassian.net/browse/SDK-1246
+  dfx canister call --query e2e_project_frontend http_request '(record{url="/test_alias_file";headers=vec{};method="GET";body=vec{}})'
+  # output looks like this:
+  #     (
+  #       record {
+  #         body = blob "test alias file\0a";
+  #         headers = vec {
+  #           record {
+  #             "etag";
+  #             "\"67fb58e3ea4556105dd5a4080e8d386e0c5f9bf5f505dd0f4a2bd865ec27c606\"";
+  #           };
+  #           record { "content-type"; "text/html" };
+  #           record {
+  #             "IC-Certificate";
+  #             "certificate=:2dn3omR0cmVlgwGDAYMBgwJIY2FuaXN0ZXKDAYIEWCB6TamgcIBRRI9+Lfe6n1mQhfqXmFKDoOihriXzPhQ8YIMBgwJKgAAAAAAQAAIBAYMBgwGDAk5jZXJ0aWZpZWRfZGF0YYIDWCDclbSRbugItvQEwCwQDt2dGNnmSWXaDfyenwBDFGQev4IEWCDN7xKU70bQDnAGR8er6+SatQSi07lHyqye6YJDNdfzOYIEWCB326K/dXJismN6beQyTC9LeKBpzbOmOZPkRjpu4uIkt4IEWCCsjlsVa3dA4Inoj23KaUsHLZTOoZT/Uqzog9gmkovo34IEWCB9LBGrUHBg7qBYleBnVlVc672alvi9zWYf8D/2CKvHMoIEWCCMpFgCT+gbsEX08W0sB8h46ysJg+clCHOjW/qmKqWCYIMBggRYIIUC8ZdiVnT/LeyjAp84wEB9JBYPO1U4tZCYvNVw1eM3gwJEdGltZYIDScD5rc7w2aTEF2lzaWduYXR1cmVYMIQdy8BFvQjET2yjcKYQ47d62tiCtV4lfcjNiyDYWGM3FXOgjORzH5MnoSIY1HwJrA==:, tree=:2dn3gwGDAktodHRwX2Fzc2V0c4MBggRYINCuA0oUZ982kzKVIIrBrs5i693coETpgd/s0JEVMozsgwGDAlAvdGVzdF9hbGlhc19maWxlggNYIGf7WOPqRVYQXdWkCA6NOG4MX5v19QXdD0or2GXsJ8YGggRYICTVHJF+Vk5ccI2w7iQhhyiMkcffMLtaKScYnwtlBC+BggRYIFce2lClRDSsnfeLc7YS3j0JsELBY5a3bbUPh+luB7dD:";
+  #           };
+  #         };
+  #         streaming_strategy = null;
+  #         status_code = 200 : nat16;
+  #       },
+  #     )
+
+
   assert_command dfx canister call e2e_project_frontend set_asset_properties '( record { key="/test_alias_file.html"; is_aliased=opt(opt(false))  })'
+
+  dfx canister call --query e2e_project_frontend http_request '(record{url="/test_alias_file";headers=vec{};method="GET";body=vec{}})'
+  # output looks like this.  Notice that the body and status code have changed, but the certificate is exactly the same.
+  #   (
+  #     record {
+  #       body = blob "not found";
+  #       headers = vec {
+  #         record { "content-type"; "text/plain" };
+  #         record {
+  #           "IC-Certificate";
+  #           "certificate=:2dn3omR0cmVlgwGDAYMBgwJIY2FuaXN0ZXKDAYIEWCB6TamgcIBRRI9+Lfe6n1mQhfqXmFKDoOihriXzPhQ8YIMBgwJKgAAAAAAQAAIBAYMBgwGDAk5jZXJ0aWZpZWRfZGF0YYIDWCDclbSRbugItvQEwCwQDt2dGNnmSWXaDfyenwBDFGQev4IEWCDN7xKU70bQDnAGR8er6+SatQSi07lHyqye6YJDNdfzOYIEWCB326K/dXJismN6beQyTC9LeKBpzbOmOZPkRjpu4uIkt4IEWCCsjlsVa3dA4Inoj23KaUsHLZTOoZT/Uqzog9gmkovo34IEWCCoBmzNJBeJUuPksGWjOP0JgQStmSri6XGg+//B8CS+PoIEWCApbis9cAc4Tv7tS97Vk+Dc14EE/styzggrRvkalYlD1YMBggRYIDG3uAnEPmC4tIPj0xDqUySASclS7unWnd6oG+IqIqGzgwJEdGltZYIDSaiiuufz2aTEF2lzaWduYXR1cmVYMLMbF+o1s///LvhWJ4sylt9/07OokAlX8L6+Dfd8L2KDUGgCzuPvmIy/3NC0+LMpEw==:, tree=:2dn3gwGDAktodHRwX2Fzc2V0c4MBggRYINCuA0oUZ982kzKVIIrBrs5i693coETpgd/s0JEVMozsgwGDAlAvdGVzdF9hbGlhc19maWxlggNYIGf7WOPqRVYQXdWkCA6NOG4MX5v19QXdD0or2GXsJ8YGggRYICTVHJF+Vk5ccI2w7iQhhyiMkcffMLtaKScYnwtlBC+BggRYIFce2lClRDSsnfeLc7YS3j0JsELBY5a3bbUPh+luB7dD:";
+  #         };
+  #       };
+  #       streaming_strategy = null;
+  #       status_code = 404 : nat16;
+  #     },
+  #   )
+
   assert_command_fail curl --fail -vv http://localhost:"$PORT"/test_alias_file?canisterId="$ID"
-  assert_match "404" "$stderr"
+
+  # this should succeed, with output 404, like so:
+  # assert_match "404" "$stderr"
+
+  # However, due to returning the wrong certificate, it fails with Err(InvalidResponseHashes)
+  # see https://dfinity.atlassian.net/browse/SDK-1246
+  assert_contains "500 Internal Server Error"
+
+
   assert_command dfx canister call e2e_project_frontend set_asset_properties '( record { key="/test_alias_file.html"; is_aliased=opt(opt(true))  })'
   assert_command curl --fail -vv http://localhost:"$PORT"/test_alias_file?canisterId="$ID"
   assert_match "200 OK" "$stderr"
@@ -1315,7 +1434,11 @@ CHERRIES" "$stdout"
   assert_match "200 OK" "$stderr"
   assert_match "test alias file"
   assert_command_fail curl --fail -vv http://localhost:"$PORT"/test_alias_file?canisterId="$ID"
-  assert_match "404 Not Found" "$stderr"
+
+  # again see # see https://dfinity.atlassian.net/browse/SDK-1246, this should be 404
+  # assert_match "404 Not Found" "$stderr"
+  assert_contains "500 Internal Server Error"
+
   assert_command curl --fail -vv http://localhost:"$PORT"/index_test?canisterId="$ID"
   assert_match "200 OK" "$stderr"
   assert_match "test index file"

diff --git a/nix/sources.json b/nix/sources.json
@@ -142,21 +142,19 @@
     },
     "icx-proxy-x86_64-darwin": {
         "builtin": false,
-        "rev": "594b6c81cde6da4e08faee8aa8e5a2e6ae815602",
-        "sha256": "09rxh6kjwy7qfsvvsy6xjzyn4r4zlb78k1qipi4k3x0w0ajvp0sp",
-        "tag": "rev-c312760",
+        "rev": "91bf38ff3cb927cb94027d9da513cd15f91a5b04",
+        "sha256": "09y3q0ri294qkfp5bywpm37f7c7gd44xq8a2fx5lafzfd98xb6v1",
         "type": "file",
-        "url": "https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-macos.tar.gz",
-        "url_template": "https://github.com/dfinity/icx-proxy/releases/download/<tag>/binaries-macos.tar.gz"
+        "url": "https://download.dfinity.systems/ic/91bf38ff3cb927cb94027d9da513cd15f91a5b04/openssl-static-binaries/x86_64-darwin/icx-proxy-dev.gz",
+        "url_template": "https://download.dfinity.systems/ic/<rev>/openssl-static-binaries/x86_64-darwin/icx-proxy-dev.gz"
     },
     "icx-proxy-x86_64-linux": {
         "builtin": false,
-        "rev": "594b6c81cde6da4e08faee8aa8e5a2e6ae815602",
-        "sha256": "18czg11v5hiczqrahr962wmjig3gcafplqiprlnx44kmzfhi4mks",
-        "tag": "rev-c312760",
+        "rev": "91bf38ff3cb927cb94027d9da513cd15f91a5b04",
+        "sha256": "0h8k4hf42grjhhs6nnl8gcb9k02cs4dm1a25mjlkf6vi3f6sx307",
         "type": "file",
-        "url": "https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-linux.tar.gz",
-        "url_template": "https://github.com/dfinity/icx-proxy/releases/download/<tag>/binaries-linux.tar.gz"
+        "url": "https://download.dfinity.systems/ic/91bf38ff3cb927cb94027d9da513cd15f91a5b04/openssl-static-binaries/x86_64-linux/icx-proxy-dev.gz",
+        "url_template": "https://download.dfinity.systems/ic/<rev>/openssl-static-binaries/x86_64-linux/icx-proxy-dev.gz"
     },
     "motoko-base": {
         "builtin": false,

diff --git a/scripts/update-replica.sh b/scripts/update-replica.sh
@@ -20,6 +20,8 @@ niv update ic-nns-init-x86_64-darwin -a rev=$SHA
 niv update ic-nns-init-x86_64-linux -a rev=$SHA
 niv update ic-starter-x86_64-darwin -a rev=$SHA
 niv update ic-starter-x86_64-linux -a rev=$SHA
+niv update icx-proxy-x86_64-darwin -a rev=$SHA
+niv update icx-proxy-x86_64-linux -a rev=$SHA
 niv update replica-x86_64-darwin -a rev=$SHA
 niv update replica-x86_64-linux -a rev=$SHA
 niv update canister_sandbox-x86_64-darwin -a rev=$SHA

diff --git a/src/dfx/assets/dfx-asset-sources.toml b/src/dfx/assets/dfx-asset-sources.toml
@@ -6,8 +6,8 @@ url = 'https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-a9f73dba-x86_64-darw
 sha256 = 'e1c1694579f46d544aa87f6387d7e5a4b096fe65015b1609a459efcbaf15890f'
 
 [x86_64-darwin.icx-proxy]
-url = 'https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-macos.tar.gz'
-sha256 = '5783bba5021cf43149bc118789cea29f6462fd97dd78bdb776f8782ea7813d27'
+url = 'https://download.dfinity.systems/ic/91bf38ff3cb927cb94027d9da513cd15f91a5b04/openssl-static-binaries/x86_64-darwin/icx-proxy-dev.gz'
+sha256 = '619bd5516aee3b454b774221dc0969efb0e3cea897fb55ae9b98241133c0c327'
 
 [x86_64-darwin.ic-admin]
 url = 'https://download.dfinity.systems/ic/91bf38ff3cb927cb94027d9da513cd15f91a5b04/openssl-static-binaries/x86_64-darwin/ic-admin.gz'
@@ -64,8 +64,8 @@ url = 'https://download.dfinity.systems/ic-ref/ic-ref-0.0.1-a9f73dba-x86_64-linu
 sha256 = '5c4967764e87d1b2945b1db027422633b48f08cd01d96ba2f622753fcb62c2a4'
 
 [x86_64-linux.icx-proxy]
-url = 'https://github.com/dfinity/icx-proxy/releases/download/rev-c312760/binaries-linux.tar.gz'
-sha256 = '7a5612a1fb7512d22dcd37627a9d626fbc282b172665a832fe2cc2b243789fa1'
+url = 'https://download.dfinity.systems/ic/91bf38ff3cb927cb94027d9da513cd15f91a5b04/openssl-static-binaries/x86_64-linux/icx-proxy-dev.gz'
+sha256 = '078cae8d1b711b37a9ac45a8501bd14c8099167b885a6b3484323f411c241341'
 
 [x86_64-linux.ic-admin]
 url = 'https://download.dfinity.systems/ic/91bf38ff3cb927cb94027d9da513cd15f91a5b04/openssl-static-binaries/x86_64-linux/ic-admin.gz'

diff --git a/src/dfx/assets/prepare_assets.rs b/src/dfx/assets/prepare_assets.rs
@@ -113,7 +113,7 @@ fn write_binary_cache(
         Compression::new(6),
     ));
     for (path, bin) in bins.into_iter().chain(
-        ["icx-proxy", "ic-ref", "moc", "mo-doc", "mo-ide"]
+        ["ic-ref", "moc", "mo-doc", "mo-ide"]
             .map(|bin| (bin.into(), bin_tars.remove(Path::new(bin)).unwrap())),
     ) {
         let mut header = Header::new_gnu();
@@ -182,6 +182,7 @@ async fn download_binaries(
         "ic-btc-adapter",
         "ic-https-outcalls-adapter",
         "ic-nns-init",
+        "icx-proxy",
         "replica",
         "canister_sandbox",
         "sandbox_launcher",
@@ -219,13 +220,13 @@ async fn download_bin_tarballs(
     sources: Arc<HashMap<String, Source>>,
 ) -> HashMap<PathBuf, Bytes> {
     let mut map = HashMap::new();
-    let [motoko, icx_proxy, ic_ref] = ["motoko", "icx-proxy", "ic-ref"].map(|pkg| {
+    let [motoko, ic_ref] = ["motoko", "ic-ref"].map(|pkg| {
         let client = client.clone();
         let source = sources[pkg].clone();
         spawn(download_and_check_sha(client, source))
     });
-    let (motoko, icx_proxy, ic_ref) = tokio::try_join!(motoko, icx_proxy, ic_ref).unwrap();
-    for tar in [motoko, icx_proxy, ic_ref] {
+    let (motoko, ic_ref) = tokio::try_join!(motoko, ic_ref).unwrap();
+    for tar in [motoko, ic_ref] {
         tar_xzf(&tar, |path, content| {
             map.insert(path, content);
         });