Merge branch 'only_enable_runtime_limits_during_js_exec' of https://g…

…ithub.com/achamayou/CCF into only_enable_runtime_limits_during_js_exec

.github/workflows/tlaplus.yml

@@ -10,8 +10,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  model-checking:
-    name: Model Checking
+  model-checking-consensus:
+    name: Model Checking - Consensus
     runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
     container:
       image: ccfmsrc.azurecr.io/ccf/ci:05-09-2023-virtual-clang15
@@ -42,8 +42,8 @@ jobs:
             tla/*.trace.tla
             tla/*.json
 
-  simulation:
-    name: Simulation
+  simulation-consensus:
+    name: Simulation - Consensus
     runs-on: ubuntu-latest
 
     steps:
@@ -63,3 +63,100 @@ jobs:
           path: |
             tla/*.trace.tla
             tla/*.json
+
+  model-checking-consistency:
+    name: Model Checking - Consistency
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+    container:
+      image: ccfmsrc.azurecr.io/ccf/ci:05-09-2023-virtual-clang15
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          sudo apt update
+          sudo apt install -y default-jre
+          python3 ./tla/install_deps.py
+
+      - name: consistency/MCSingleNode.cfg
+        run: |
+          cd tla/
+          ./tlc.sh -workers auto consistency/MCSingleNode.tla -dumpTrace json MCSingleNode.json
+
+      - name: consistency/MCSingleNodeReads.cfg
+        run: |
+          cd tla/
+          ./tlc.sh -workers auto consistency/MCSingleNodeReads.tla -dumpTrace json MCSingleNodeReads.json
+
+      - name: consistency/MCMultiNode.cfg
+        run: |
+          cd tla/
+          ./tlc.sh -workers auto consistency/MCMultiNode.tla -dumpTrace json MCMultiNode.json
+
+      - name: consistency/MCMultiNodeReads.cfg
+        run: |
+          cd tla/
+          ./tlc.sh -workers auto consistency/MCMultiNodeReads.tla -dumpTrace json MCMultiNodeReads.json
+
+      - name: consistency/MCMultiNodeReadsAlt.cfg
+        run: |
+          cd tla/
+          ./tlc.sh -workers auto consistency/MCMultiNodeReadsAlt.tla -dumpTrace json MCMultiNodeReadsAlt.json
+
+      - name: Upload TLC's out file as an artifact. Can be imported into the TLA+ Toolbox.
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: tlc
+          path: |
+            tla/consistency/*_TTrace_*.tla
+            tla/*.json
+
+  counterexamples-consistency:
+    name: Counterexamples - Consistency
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: python3 ./tla/install_deps.py
+
+      - name: consistency/MCSingleNodeCommitReachability.cfg
+        run: |
+          cd tla/
+          ./tlc_debug.sh -workers auto -config consistency/MCSingleNodeCommitReachability.cfg consistency/MCSingleNodeReads.tla
+
+      - name: consistency/MCMultiNodeCommitReachability.cfg
+        run: |
+          cd tla/
+          ./tlc_debug.sh -workers auto -config consistency/MCMultiNodeCommitReachability.cfg consistency/MCMultiNodeReads.tla
+
+      - name: consistency/MCMultiNodeInvalidReachability.cfg
+        run: |
+          cd tla/
+          ./tlc_debug.sh -workers auto -config consistency/MCMultiNodeInvalidReachability.cfg consistency/MCMultiNodeReads.tla
+
+      - name: consistency/MCMultiNodeReadsNotLinearizable.cfg
+        run: |
+          cd tla/
+          ./tlc_debug.sh -workers auto -config consistency/MCMultiNodeReadsNotLinearizable.cfg consistency/MCMultiNodeReads.tla
+
+  simulation-consistency:
+    name: Simulation - Consistency
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: python3 ./tla/install_deps.py
+
+      - name: consistency/MultiNodeReads.cfg
+        run: |
+          cd tla/
+          ./tlc.sh -workers auto -simulate num=5 -depth 50 consistency/MultiNodeReads.tla -dumpTrace json MultiNodeReads.json
+
+      - name: Upload traces in TLA and JSON format
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: tlc
+          path: |
+            tla/consistency/*_TTrace_*.tla
+            tla/*.json

tests/infra/consortium.py

@@ -522,9 +522,7 @@ def assert_service_identity(self, remote_node, service_cert_path):
         proposal = self.get_any_active_member().propose(remote_node, proposal_body)
         return self.vote_using_majority(remote_node, proposal, careful_vote)
 
-    def set_js_app_from_dir(
-        self, remote_node, bundle_path, disable_bytecode_cache=False
-    ):
+    def read_bundle_from_dir(self, bundle_path):
         if os.path.isfile(bundle_path):
             tmp_dir = tempfile.TemporaryDirectory(prefix="ccf")
             shutil.unpack_archive(bundle_path, tmp_dir.name)
@@ -547,21 +545,20 @@ def set_js_app_from_dir(
                         f"{method} {url}: module '{module_path}' not found in bundle"
                     )
 
-        proposal_body, careful_vote = self.make_proposal(
-            "set_js_app",
-            bundle={"metadata": metadata, "modules": modules},
-            disable_bytecode_cache=disable_bytecode_cache,
-        )
-        proposal = self.get_any_active_member().propose(remote_node, proposal_body)
-        # Large apps take a long time to process - wait longer than normal for commit
-        return self.vote_using_majority(remote_node, proposal, careful_vote, timeout=30)
+        return {"metadata": metadata, "modules": modules}
 
-    def set_js_app_from_json(
-        self, remote_node, json_path, disable_bytecode_cache=False
+    def set_js_app_from_dir(
+        self, remote_node, bundle_path, disable_bytecode_cache=False
     ):
+        bundle = self.read_bundle_from_dir(bundle_path)
+        return self.set_js_app_from_bundle(
+            remote_node, bundle, disable_bytecode_cache=disable_bytecode_cache
+        )
+
+    def set_js_app_from_bundle(self, remote_node, bundle, disable_bytecode_cache=False):
         proposal_body, careful_vote = self.make_proposal(
             "set_js_app",
-            bundle=slurp_json(json_path),
+            bundle=bundle,
             disable_bytecode_cache=disable_bytecode_cache,
         )
 

tests/js-custom-authorization/custom_authorization.py

@@ -16,6 +16,7 @@
 import infra.jwt_issuer
 import datetime
 import re
+import uuid
 from http import HTTPStatus
 import subprocess
 from contextlib import contextmanager
@@ -960,6 +961,12 @@ def was_cached(response):
         expect_much_smaller(repeat2, baseline)
         expect_much_smaller(repeat3, baseline)
 
+    return network
+
+
+def test_caching_of_kv_handles(network, args):
+    primary, _ = network.find_nodes()
+    with primary.client() as c:
         LOG.info("Testing caching of KV handles")
         r = c.post("/app/increment")
         assert r.status_code == http.HTTPStatus.OK, r
@@ -975,6 +982,33 @@ def was_cached(response):
     return network
 
 
+def test_caching_of_app_code(network, args):
+    primary, backups = network.find_nodes()
+    LOG.info(
+        "Testing that interpreter reuse does not persist functions past app update"
+    )
+
+    def set_app_with_placeholder(new_val):
+        LOG.info(f"Replacing placeholder with {new_val}")
+        bundle = network.consortium.read_bundle_from_dir(args.js_app_bundle)
+        # Replace placeholder blindly, in the raw bundle JSON as string
+        s = json.dumps(bundle).replace("<func_caching_placeholder>", new_val)
+        bundle = json.loads(s)
+        return network.consortium.set_js_app_from_bundle(primary, bundle)
+
+    for _ in range(5):
+        v = str(uuid.uuid4())
+        p = set_app_with_placeholder(v)
+        for node in [primary, *backups]:
+            with node.client() as c:
+                infra.commit.wait_for_commit(client=c, view=p.view, seqno=p.seqno)
+                r = c.get("/app/func_caching")
+                assert r.status_code == http.HTTPStatus.OK, r
+                assert r.body.text() == v
+
+    return network
+
+
 def run_interpreter_reuse(args):
     # The js_app_bundle arg includes TS and Node dependencies, so must be built here
     # before deploying (and then we deploy the produces /dist folder)
@@ -989,7 +1023,9 @@ def run_interpreter_reuse(args):
     ) as network:
         network.start_and_open(args)
 
-        network = test_reused_interpreter_behaviour(network, args)
+        network = test_reused_interpreter_behaviour(network, args)  #
+        network = test_caching_of_kv_handles(network, args)
+        network = test_caching_of_app_code(network, args)
 
 
 if __name__ == "__main__":
@@ -1035,7 +1071,7 @@ def run_interpreter_reuse(args):
     cr.add(
         "interpreter_reuse",
         run_interpreter_reuse,
-        nodes=infra.e2e_args.nodes(cr.args, 1),
+        nodes=infra.e2e_args.min_nodes(cr.args, f=1),
         js_app_bundle=os.path.join(cr.args.js_app_bundle, "js-interpreter-reuse"),
     )
 

tests/js-interpreter-reuse/app.json

@@ -74,6 +74,19 @@
           "key": "increment"
         }
       }
+    },
+    "/func_caching": {
+      "get": {
+        "js_module": "func_caching.js",
+        "js_function": "func_caching",
+        "forwarding_required": "never",
+        "authn_policies": ["no_auth"],
+        "mode": "readonly",
+        "openapi": {},
+        "interpreter_reuse": {
+          "key": "func_caching"
+        }
+      }
     }
   }
 }

tests/js-interpreter-reuse/src/func_caching.js

@@ -0,0 +1,6 @@
+export function func_caching(request) {
+  const s = "<func_caching_placeholder>";
+  console.log(`Executing with s: ${s}`);
+
+  return { body: s };
+}

tests/js-interpreter-reuse/src/rollup_entry.ts

@@ -1,3 +1,4 @@
 export * from "./cache.js";
+export * from "./func_caching.js";
 export * from "./di_sample";
 export * from "./global_handle";

tests/js-modules/modules.py

@@ -483,7 +483,8 @@ def test_npm_app(network, args):
     bundle_path = os.path.join(
         app_dir, "dist", "bundle.json"
     )  # Produced by build step of test npm-app
-    network.consortium.set_js_app_from_json(primary, bundle_path)
+    bundle = infra.consortium.slurp_json(bundle_path)
+    network.consortium.set_js_app_from_bundle(primary, bundle)
 
     LOG.info("Calling npm app endpoints")
     with primary.client("user0") as c:
@@ -1211,7 +1212,8 @@ def test_js_execution_time(network, args):
     bundle_path = os.path.join(
         app_dir, "dist", "bundle.json"
     )  # Produced by build step of test npm-app in the previous test_npm_app
-    network.consortium.set_js_app_from_json(primary, bundle_path)
+    bundle = infra.consortium.slurp_json(bundle_path)
+    network.consortium.set_js_app_from_bundle(primary, bundle)
 
     LOG.info("Store JWT signing keys")
     jwt_key_priv_pem, _ = infra.crypto.generate_rsa_keypair(2048)