Skip to content

Commit

Permalink
fix test
Browse files Browse the repository at this point in the history
  • Loading branch information
@chenyan-dfinity
chenyan-dfinity committed Sep 15, 2023
1 parent d455af2 commit 8c77784
Show file tree
Hide file tree
Showing 6 changed files with 220 additions and 61 deletions.
84 changes: 84 additions & 0 deletions e2e/assets/playground_backend/service/pool/Logs.mo
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,88 @@
import Map "mo:base/RBTree";
import {compare} "mo:base/Text";
import {toArray} "mo:base/Iter";
import {now = timeNow} "mo:base/Time";
import {toText} "mo:base/Int";
import {get} "mo:base/Option";

module {
public type Origin = { origin: Text; tags: [Text] };
public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>);
public class StatsByOrigin() {
var canisters = Map.RBTree<Text, Nat>(compare);
var installs = Map.RBTree<Text, Nat>(compare);
public func share() : SharedStatsByOrigin = (canisters.share(), installs.share());
public func unshare(x : SharedStatsByOrigin) {
canisters.unshare(x.0);
installs.unshare(x.1);
};
func addTags(map: Map.RBTree<Text,Nat>, list: [Text]) {
for (tag in list.vals()) {
switch (map.get(tag)) {
case null { map.put(tag, 1) };
case (?n) { map.put(tag, n + 1) };
};
};
};
// if to is null, delete the from tag
func merge_tag_(map: Map.RBTree<Text,Nat>, from: Text, opt_to: ?Text) {
ignore do ? {
let n1 = map.remove(from)!;
let to = opt_to!;
switch (map.get(to)) {
case null { map.put(to, n1) };
case (?n2) { map.put(to, n1 + n2) };
};
};
};
public func merge_tag(from: Text, to: ?Text) {
merge_tag_(canisters, from, to);
merge_tag_(installs, from, to);
};
public func addCanister(origin: Origin) {
addTags(canisters, ["origin:" # origin.origin]);
addTags(canisters, origin.tags);
};
public func addInstall(origin: Origin) {
addTags(installs, ["origin:" # origin.origin]);
addTags(installs, origin.tags);
};
public func dump() : ([(Text, Nat)], [(Text, Nat)]) {
(toArray<(Text, Nat)>(canisters.entries()),
toArray<(Text, Nat)>(installs.entries()),
)
};
public func metrics() : Text {
var result = "";
let now = timeNow() / 1_000_000;
let canister_playground = get(canisters.get("origin:playground"), 0);
let canister_dfx = get(canisters.get("origin:dfx"), 0);
let install_playground = get(installs.get("origin:playground"), 0);
let install_dfx = get(installs.get("origin:dfx"), 0);
let profiling = get(installs.get("wasm:profiling"), 0);
let asset = get(installs.get("wasm:asset"), 0);
let install = get(installs.get("mode:install"), 0);
let reinstall = get(installs.get("mode:reinstall"), 0);
let upgrade = get(installs.get("mode:upgrade"), 0);
result := result
# encode_single_value("counter", "create_from_playground", canister_playground, "Number of canisters created from playground", now)
# encode_single_value("counter", "install_from_playground", install_playground, "Number of Wasms installed from playground", now)
# encode_single_value("counter", "create_from_dfx", canister_dfx, "Number of canisters created from dfx", now)
# encode_single_value("counter", "install_from_dfx", install_dfx, "Number of Wasms installed from dfx", now)
# encode_single_value("counter", "profiling", profiling, "Number of Wasms profiled", now)
# encode_single_value("counter", "asset", asset, "Number of asset Wasms canister installed", now)
# encode_single_value("counter", "install", install, "Number of Wasms with install mode", now)
# encode_single_value("counter", "reinstall", reinstall, "Number of Wasms with reinstall mode", now)
# encode_single_value("counter", "upgrade", upgrade, "Number of Wasms with upgrad mode", now);
result;
};
};
public func encode_single_value(kind: Text, name: Text, number: Int, desc: Text, time: Int) : Text {
"# HELP " # name # " " # desc # "\n" #
"# TYPE " # name # " " # kind # "\n" #
name # " " # toText(number) # " " # toText(time) # "\n"
};

public type Stats = {
num_of_canisters: Nat;
num_of_installs: Nat;
Expand Down
109 changes: 88 additions & 21 deletions e2e/assets/playground_backend/service/pool/Main.mo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import Option "mo:base/Option";
import Nat "mo:base/Nat";
import Text "mo:base/Text";
import Array "mo:base/Array";
import Buffer "mo:base/Buffer";
import List "mo:base/List";
import Deque "mo:base/Deque";
import Result "mo:base/Result";
Expand All @@ -17,29 +18,31 @@ import PoW "./PoW";
import Logs "./Logs";
import Metrics "./Metrics";
import Wasm "canister:wasm-utils";
import Blob "mo:base/Blob";
import Buffer "mo:base/Buffer";
import Nat32 "mo:base/Nat32";

shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
let IC : ICType.Self = actor "aaaaa-aa";
let params = Option.get(opt_params, Types.defaultParams);
var pool = Types.CanisterPool(params.max_num_canisters, params.canister_time_to_live, params.max_family_tree_size);
let nonceCache = PoW.NonceCache(params.nonce_time_to_live);
var statsByOrigin = Logs.StatsByOrigin();

stable let controller = creator.caller;
stable var stats = Logs.defaultStats;
stable var stablePool : [Types.CanisterInfo] = [];
stable var stableMetadata : [(Principal, (Int, Bool))] = [];
stable var stableChildren : [(Principal, [Principal])] = [];
stable var stableTimers : [Types.CanisterInfo] = [];
stable var previousParam : ?Types.InitParams = null;
stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf);

system func preupgrade() {
let (tree, metadata, children) = pool.share();
let (tree, metadata, children, timers) = pool.share();
stablePool := tree;
stableMetadata := metadata;
stableChildren := children;
stableTimers := timers;
previousParam := ?params;
stableStatsByOrigin := statsByOrigin.share();
};

system func postupgrade() {
Expand All @@ -49,14 +52,19 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
};
pool.unshare(stablePool, stableMetadata, stableChildren);
for (info in stableTimers.vals()) {
updateTimer(info);
};
statsByOrigin.unshare(stableStatsByOrigin);
};

public query func getInitParams() : async Types.InitParams {
params;
};

public query func getStats() : async Logs.Stats {
stats;
public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)]) {
let (canister, install) = statsByOrigin.dump();
(stats, canister, install);
};

public query func balance() : async Nat {
Expand All @@ -68,7 +76,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
ignore Cycles.accept amount;
};

private func getExpiredCanisterInfo() : async Types.CanisterInfo {
private func getExpiredCanisterInfo(origin : Logs.Origin) : async Types.CanisterInfo {
switch (pool.getExpiredCanisterId()) {
case (#newId) {
Cycles.add(params.cycles_per_canister);
Expand All @@ -77,6 +85,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
let info = { id = cid.canister_id; timestamp = now };
pool.add info;
stats := Logs.updateStats(stats, #getId(params.cycles_per_canister));
statsByOrigin.addCanister(origin);
info;
};
case (#reuse info) {
Expand All @@ -89,15 +98,17 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
Cycles.add topUpCycles;
await IC.deposit_cycles cid;
};
// Lazily cleanup the reused canister
await IC.uninstall_code cid;
if (Option.isSome(status.module_hash)) {
await IC.uninstall_code cid;
};
switch (status.status) {
case (#stopped or #stopping) {
await IC.start_canister cid;
};
case _ {};
};
stats := Logs.updateStats(stats, #getId topUpCycles);
statsByOrigin.addCanister(origin);
info;
};
case (#outOfCapacity time) {
Expand All @@ -107,8 +118,23 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
};
};
func validateOrigin(origin: Logs.Origin) : Bool {
if (origin.origin == "") {
return false;
};
for (tag in origin.tags.vals()) {
// reject server side tags
if (tag == "mode:install" or tag == "mode:reinstall" or tag == "mode:upgrade" or tag == "wasm:profiling" or tag == "wasm:asset") {
return false;
}
};
return true;
};

public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce) : async Types.CanisterInfo {
public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce, origin : Logs.Origin) : async Types.CanisterInfo {
if (not validateOrigin(origin)) {
throw Error.reject "Please specify a valid origin";
};
if (caller != controller and not nonceCache.checkProofOfWork(nonce)) {
stats := Logs.updateStats(stats, #mismatch);
throw Error.reject "Proof of work check failed";
Expand All @@ -119,10 +145,14 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
throw Error.reject "Nonce already used";
};
nonceCache.add nonce;
await getExpiredCanisterInfo();
await getExpiredCanisterInfo(origin);
};

public shared ({ caller }) func installCode(info : Types.CanisterInfo, args : Types.InstallArgs, profiling : Bool, is_whitelisted : Bool) : async Types.CanisterInfo {
type InstallConfig = { profiling: Bool; is_whitelisted: Bool; origin: Logs.Origin };
public shared ({ caller }) func installCode(info : Types.CanisterInfo, args : Types.InstallArgs, install_config : InstallConfig) : async Types.CanisterInfo {
if (not validateOrigin(install_config.origin)) {
throw Error.reject "Please specify a valid origin";
};
if (info.timestamp == 0) {
stats := Logs.updateStats(stats, #mismatch);
throw Error.reject "Cannot install removed canister";
Expand All @@ -132,14 +162,14 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
throw Error.reject "Cannot find canister";
} else {
let config = {
profiling;
profiling = install_config.profiling;
remove_cycles_add = true;
limit_stable_memory_page = ?(16384 : Nat32); // Limit to 1G of stable memory
backend_canister_id = ?Principal.fromActor(this);
};
let wasm = if (caller == controller) {
let wasm = if (caller == controller and install_config.is_whitelisted) {
args.wasm_module;
} else if (is_whitelisted) {
} else if (install_config.is_whitelisted) {
await Wasm.is_whitelisted(args.wasm_module);
} else {
await Wasm.transform(args.wasm_module, config);
Expand All @@ -152,13 +182,42 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
await IC.install_code newArgs;
stats := Logs.updateStats(stats, #install);
switch (pool.refresh(info, profiling)) {
case (?newInfo) newInfo;

// Build tags from install arguments
let tags = Buffer.fromArray<Text>(install_config.origin.tags);
if (install_config.profiling) {
tags.add("wasm:profiling");
};
if (install_config.is_whitelisted) {
tags.add("wasm:asset");
};
switch (args.mode) {
case (#install) { tags.add("mode:install") };
case (#upgrade) { tags.add("mode:upgrade") };
case (#reinstall) { tags.add("mode:reinstall") };
};
let origin = { origin = install_config.origin.origin; tags = Buffer.toArray(tags) };
statsByOrigin.addInstall(origin);
switch (pool.refresh(info, install_config.profiling)) {
case (?newInfo) {
updateTimer(newInfo);
newInfo;
};
case null { throw Error.reject "Cannot find canister" };
};
};
};

func updateTimer(info: Types.CanisterInfo) {
func job() : async () {
pool.removeTimer(info.id);
// It is important that the timer job checks for the timestamp first.
// This prevents late-runner jobs from deleting newly installed code.
await removeCode(info);
};
pool.updateTimer(info, job);
};

public func callForward(info : Types.CanisterInfo, function : Text, args : Blob) : async Blob {
if (pool.find info) {
await InternetComputer.call(info.id, function, args);
Expand Down Expand Up @@ -225,12 +284,19 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
throw Error.reject "Only called by controller";
};
stats := Logs.defaultStats;
statsByOrigin := Logs.StatsByOrigin();
};
public shared ({ caller }) func mergeTags(from: Text, to: ?Text) : async () {
if (caller != controller) {
throw Error.reject "Only called by controller";
};
statsByOrigin.merge_tag(from, to);
};

// Metrics
public query func http_request(req : Metrics.HttpRequest) : async Metrics.HttpResponse {
if (req.url == "/metrics") {
let body = Metrics.metrics stats;
let body = Metrics.metrics(stats);
{
status_code = 200;
headers = [("Content-Type", "text/plain; version=0.0.4"), ("Content-Length", Nat.toText(body.size()))];
Expand Down Expand Up @@ -278,7 +344,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
if (not pool.findId caller) {
throw Error.reject "Only a canister managed by the Motoko Playground can call create_canister";
};
let info = await getExpiredCanisterInfo();
let info = await getExpiredCanisterInfo({origin="spawned"; tags=[]});
let result = pool.setChild(caller, info.id);
if (not result) {
throw Error.reject("In the Motoko Playground, each top level canister can only spawn " # Nat.toText(params.max_family_tree_size) # " descendants including itself");
Expand All @@ -299,12 +365,12 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
wasm_module : ICType.wasm_module;
mode : { #reinstall; #upgrade; #install };
canister_id : ICType.canister_id;
is_whitelisted : Bool;
}) : async () {
switch (sanitizeInputs(caller, canister_id)) {
case (#ok info) {
let args = { arg; wasm_module; mode; canister_id };
ignore await installCode(info, args, pool.profiling caller, is_whitelisted); // inherit the profiling of the parent
let config = { profiling = pool.profiling caller; is_whitelisted = false; origin = {origin = "spawned"; tags = [] } };
ignore await installCode(info, args, config); // inherit the profiling of the parent
};
case (#err makeMsg) throw Error.reject(makeMsg "install_code");
};
Expand Down Expand Up @@ -379,6 +445,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
#installCode : Any;
#removeCode : Any;
#resetStats : Any;
#mergeTags : Any;
#wallet_receive : Any;

#create_canister : Any;
Expand Down
7 changes: 1 addition & 6 deletions e2e/assets/playground_backend/service/pool/Metrics.mo
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import Text "mo:base/Text";
import Time "mo:base/Time";
import Int "mo:base/Int";
import Logs "./Logs";

module {
Expand All @@ -15,11 +14,7 @@ module {
headers: [(Text, Text)];
body: Blob;
};
func encode_single_value(kind: Text, name: Text, number: Int, desc: Text, time: Int) : Text {
"# HELP " # name # " " # desc # "\n" #
"# TYPE " # name # " " # kind # "\n" #
name # " " # Int.toText(number) # " " # Int.toText(time) # "\n"
};
let encode_single_value = Logs.encode_single_value;
public func metrics(stats: Logs.Stats) : Blob {
let now = Time.now() / 1_000_000;
var result = "";
Expand Down
Loading

0 comments on commit 8c77784

Please sign in to comment.