Add rudimentary footprint measurement for PRs (#369)

.github/workflows/ci.yml

@@ -54,3 +54,10 @@ jobs:
           restore-keys: ${{ hashFiles('rust-toolchain.toml') }}
       - run: ./scripts/setup.sh
       - run: ./scripts/ci.sh
+      - run: ./scripts/footprint.sh
+      - run: mv footprint.toml footprint-${{ github.event_name }}.toml
+      - uses: actions/upload-artifact@v4
+        if: github.event_name == 'push'
+        with:
+          name: footprint
+          path: footprint-push.toml

crates/xtask/Cargo.toml

@@ -15,6 +15,8 @@ lazy_static = "1.4.0"
 log = "0.4.20"
 probe-rs = "0.22.0"
 rustc-demangle = "0.1.23"
+serde = { version = "1.0.195", features = ["derive"] }
 sha2 = "0.10.8"
 stack-sizes = "0.5.0"
 strum = { version = "0.25.0", features = ["derive"] }
+toml = "0.8.8"

crates/xtask/src/footprint.rs

@@ -0,0 +1,102 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet};
+
+use anyhow::{ensure, Context, Result};
+use serde::{Deserialize, Serialize};
+
+use crate::{fs, output_command, wrap_command};
+
+#[derive(Serialize, Deserialize)]
+struct Footprint {
+    version: usize,
+    measurements: Vec<Measurement>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Measurement {
+    key: Vec<String>,
+    value: usize,
+}
+
+pub fn update(key: &str, value: usize) -> Result<()> {
+    const PATH: &str = "footprint.toml";
+    let mut footprint = Footprint::read(PATH)?;
+    let key = key.split_whitespace().map(|x| x.to_string()).collect();
+    let measurement = Measurement { key, value };
+    match footprint.measurements.iter_mut().find(|x| x.key == measurement.key) {
+        Some(x) => x.value = value,
+        None => footprint.measurements.push(measurement),
+    }
+    fs::write_toml(PATH, &footprint)?;
+    Ok(())
+}
+
+pub fn compare() -> Result<()> {
+    let base = Footprint::read_measurements("footprint-push.toml")?;
+    let head = Footprint::read_measurements("footprint-pull_request.toml")?;
+    let keys: HashSet<&Vec<String>> = base.keys().chain(head.keys()).collect();
+    let mut keys: Vec<&Vec<String>> = keys.into_iter().collect();
+    keys.sort();
+    let print = |k, x| match x {
+        Some(x) => print!(" {x}"),
+        None => print!(" no-{k}"),
+    };
+    for key in keys {
+        print!("{key:?}");
+        let base = base.get(key).cloned();
+        let head = head.get(key).cloned();
+        let diff = base.and_then(|base| head.map(|head| head - base));
+        print("base", base);
+        print("head", head);
+        print("diff", diff);
+        println!();
+    }
+    Ok(())
+}
+
+/// Returns the sum of the .text and .data size.
+pub fn rust_size(elf: &str) -> Result<usize> {
+    let mut size = wrap_command()?;
+    size.args(["rust-size", elf]);
+    let output = String::from_utf8(output_command(&mut size)?.stdout)?;
+    let line = output.lines().nth(1).context("parsing rust-size output")?;
+    let words =
+        line.split_whitespace().take(2).map(|x| Ok(x.parse()?)).collect::<Result<Vec<_>>>()?;
+    Ok(words.iter().sum())
+}
+
+impl Footprint {
+    fn read(path: &str) -> Result<Self> {
+        let footprint = if fs::exists(path) {
+            fs::read_toml(path)?
+        } else {
+            Footprint { version: VERSION, measurements: Vec::new() }
+        };
+        ensure!(footprint.version == VERSION);
+        Ok(footprint)
+    }
+
+    fn read_measurements(path: &str) -> Result<HashMap<Vec<String>, usize>> {
+        let mut result = HashMap::new();
+        let footprint = Self::read(path)?;
+        for Measurement { key, value } in footprint.measurements {
+            ensure!(result.insert(key, value).is_none(), "duplicate key in {path}");
+        }
+        Ok(result)
+    }
+}
+
+const VERSION: usize = 1;

crates/xtask/src/fs.rs

@@ -18,6 +18,8 @@ use std::fs::Metadata;
 use std::path::{Path, PathBuf};
 
 use anyhow::{Context, Result};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
 
 pub fn canonicalize(path: impl AsRef<Path>) -> Result<PathBuf> {
     let name = path.as_ref().display();
@@ -57,6 +59,13 @@ pub fn read(path: impl AsRef<Path>) -> Result<Vec<u8>> {
     std::fs::read(path.as_ref()).with_context(|| format!("reading {name}"))
 }
 
+pub fn read_toml<T: DeserializeOwned>(path: impl AsRef<Path>) -> Result<T> {
+    let name = path.as_ref().display();
+    let contents = read(path.as_ref())?;
+    let data = String::from_utf8(contents).with_context(|| format!("reading {name}"))?;
+    toml::from_str(&data).with_context(|| format!("parsing {name}"))
+}
+
 pub fn remove_file(path: impl AsRef<Path>) -> Result<()> {
     let name = path.as_ref().display();
     std::fs::remove_file(path.as_ref()).with_context(|| format!("removing {name}"))
@@ -76,3 +85,10 @@ pub fn write(path: impl AsRef<Path>, contents: impl AsRef<[u8]>) -> Result<()> {
     std::fs::write(path.as_ref(), contents).with_context(|| format!("writing {name}"))?;
     Ok(())
 }
+
+pub fn write_toml<T: Serialize>(path: impl AsRef<Path>, contents: &T) -> Result<()> {
+    let name = path.as_ref().display();
+    let contents = toml::to_string(contents).with_context(|| format!("displaying {name}"))?;
+    write(path.as_ref(), contents)?;
+    Ok(())
+}

crates/xtask/src/main.rs

@@ -31,6 +31,7 @@ use rustc_demangle::demangle;
 use sha2::{Digest, Sha256};
 use strum::{Display, EnumString};
 
+mod footprint;
 mod fs;
 mod lazy;
 
@@ -67,8 +68,12 @@ struct MainOptions {
     /// Prints basic size information.
     #[clap(long)]
     size: bool,
-    // TODO: Add a flag to add "-C link-arg=-Map=output.map" to get the map of why the linker
-    // added/kept something.
+
+    /// Updates footprint.toml to make the measured footprint to the provided key.
+    ///
+    /// The key is a space-separated list of strings.
+    #[clap(long)]
+    footprint: Option<String>,
 }
 
 #[derive(clap::Subcommand)]
@@ -85,6 +90,11 @@ enum MainCommand {
         #[clap(long, default_values_t = ["full-api".to_string()])]
         features: Vec<String>,
     },
+
+    /// Dumps a table comparison between footprint-base.toml and footprint-pull_request.toml.
+    ///
+    /// If footprint-base.toml is missing, it is omitted from the table.
+    Footprint,
 }
 
 #[derive(clap::Args)]
@@ -269,6 +279,7 @@ impl Flags {
                 cargo.arg(format!("--output=examples/{lang}/api.{ext}"));
                 execute_command(&mut cargo)?;
             }
+            MainCommand::Footprint => footprint::compare()?,
         }
         Ok(())
     }
@@ -364,13 +375,19 @@ impl AppletOptions {
             None => "target/wasefire/applet.wasm",
         };
         let changed = copy_if_changed(&out, applet)?;
-        if native.is_some() && main.size {
-            let mut size = wrap_command()?;
-            size.args(["rust-size", applet]);
-            let output = String::from_utf8(output_command(&mut size)?.stdout)?;
-            // We assume the interesting part is the first line after the header.
-            for line in output.lines().take(2) {
-                println!("{line}");
+        if native.is_some() {
+            if main.size {
+                let mut size = wrap_command()?;
+                size.args(["rust-size", applet]);
+                let output = String::from_utf8(output_command(&mut size)?.stdout)?;
+                // We assume the interesting part is the first line after the header.
+                for line in output.lines().take(2) {
+                    println!("{line}");
+                }
+            }
+            if let Some(key) = &main.footprint {
+                let value = footprint::rust_size(applet)?;
+                footprint::update(key, value)?;
             }
         }
         if native.is_none() && changed {
@@ -429,6 +446,9 @@ impl AppletOptions {
                 println!("Applet size (after wasm-opt): {}", fs::metadata(wasm)?.len());
             }
         }
+        if let Some(key) = &main.footprint {
+            footprint::update(key, fs::metadata(wasm)?.len() as usize)?;
+        }
         Ok(())
     }
 }
@@ -586,6 +606,10 @@ impl RunnerOptions {
             size.arg(&elf);
             execute_command(&mut size)?;
         }
+        if let Some(key) = &main.footprint {
+            let value = footprint::rust_size(&elf)?;
+            footprint::update(key, value)?;
+        }
         if let Some(stack_sizes) = self.stack_sizes {
             let elf = fs::read(&elf)?;
             let symbols = stack_sizes::analyze_executable(&elf)?;

scripts/footprint.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+. scripts/log.sh
+
+# This script computes the footprint of different applets and runners in
+# different configurations. The continuous integration will use this to show
+# footprint impact in pull requests.
+
+bool() { if [ -n "$1" ]; then echo "$2"; else echo "$3"; fi; }
+
+runner=nordic
+for applet in hello hsm; do
+  for release in '' --release; do
+    for native in '' --native-target=thumbv7em-none-eabi; do
+      config="$(bool "$release" release debug)"
+      config="$config $(bool "$native" native wasm)"
+      x cargo xtask --footprint="applet $applet $config" \
+        $release $native applet rust $applet
+      x cargo xtask --footprint="runner $runner $config" \
+        $release ${native:+--native} runner $runner
+    done
+  done
+done