From 3cda30e98b3e76686376a68cab69454087b58c63 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 7 Apr 2022 21:59:48 +0800 Subject: [PATCH 01/84] Migrated the compile phase away from legion --- Cargo.lock | 86 +++++++++++++++++++ crates/compiler/Cargo.toml | 3 + crates/compiler/src/build_context.rs | 15 +++- crates/compiler/src/codegen/components.rs | 8 +- crates/compiler/src/compile/cargo_build.rs | 34 ++------ crates/compiler/src/compile/components.rs | 2 +- crates/compiler/src/compile/mod.rs | 44 ++++++++-- .../src/compile/write_project_to_disk.rs | 3 +- crates/compiler/src/phases.rs | 16 +++- 9 files changed, 167 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4536f76600..fcdcd9bbc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,6 +170,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + [[package]] name = "bstr" version = "0.2.17" @@ -1139,6 +1148,7 @@ dependencies = [ "heck 0.4.0", "hotg-rune-core", "hotg-rune-proc-blocks", + "im", "indexmap", "indoc", "jsonschema", @@ -1149,10 +1159,12 @@ dependencies = [ "proc-macro2", "quote", "regex", + "salsa", "schemars", "serde", "serde_json", "serde_yaml", + "thiserror", "toml", "zip", ] @@ -1293,6 +1305,20 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "im" +version = "15.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "111c1983f3c5bb72732df25cddacee9b546d08325fb584b5ebd38148be7b0246" +dependencies = [ + "bitmaps", + "rand_core 0.5.1", + "rand_xoshiro", + "sized-chunks", + "typenum", + "version_check", +] + [[package]] name = "image" version = "0.23.14" @@ -2159,6 +2185,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" + [[package]] name = "rand_core" version = "0.6.3" @@ -2168,6 +2200,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xoshiro" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rayon" version = "1.5.2" @@ -2364,6 +2405,35 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "salsa" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b84d9f96071f3f3be0dc818eae3327625d8ebc95b58da37d6850724f31d3403" +dependencies = [ + "crossbeam-utils", + "indexmap", + "lock_api", + "log", + "oorandom", + "parking_lot 0.11.2", + "rustc-hash", + "salsa-macros", + "smallvec", +] + +[[package]] +name = "salsa-macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3904a4ba0a9d0211816177fd34b04c7095443f8cdacd11175064fe541c8fe2" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2554,6 +2624,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + [[package]] name = "smallvec" version = "1.8.0" @@ -2853,6 +2933,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + [[package]] name = "ucd-trie" version = "0.1.3" diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index a6f97c1eb3..3d1b400761 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -21,6 +21,7 @@ codespan-reporting = "0.11.1" heck = "0.4.0" hotg-rune-core = { path = "../rune-core", version = "^0.11.0"} hotg-rune-proc-blocks = { path = "../proc-blocks", version = "^0.11.0", default-features = false } +im = "15.0.0" indexmap = { version = "1.8.0", features = ["serde-1"] } indoc = "1.0.3" legion = { version = "0.4.0", default-features = false, features = ["serialize", "codegen", "extended-tuple-impls"] } @@ -29,10 +30,12 @@ once_cell = "1.9.0" proc-macro2 = "1.0.36" quote = "1.0.14" regex = "1.5.4" +salsa = "0.16.1" schemars = { version = "0.8.8", features = ["indexmap"] } serde = { version = "1.0.133", features = ["derive"] } serde_json = "1.0.74" serde_yaml = "0.8.23" +thiserror = "1.0.30" toml = "0.5.8" zip = "0.5.13" diff --git a/crates/compiler/src/build_context.rs b/crates/compiler/src/build_context.rs index 094ae369b7..81a2da3928 100644 --- a/crates/compiler/src/build_context.rs +++ b/crates/compiler/src/build_context.rs @@ -6,7 +6,9 @@ use std::{ use crate::codegen::RuneVersion; /// Inputs used during the compilation process. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct BuildContext { /// The name of the Rune being compiled. pub name: String, @@ -76,7 +78,14 @@ impl BuildContext { } #[derive( - Debug, Copy, Clone, PartialEq, serde::Serialize, serde::Deserialize, + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, )] pub enum Verbosity { Quiet, @@ -109,7 +118,7 @@ impl Verbosity { } /// Feature flags and other knobs that can be used during development. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct FeatureFlags { pub(crate) rune_repo_dir: Option, } diff --git a/crates/compiler/src/codegen/components.rs b/crates/compiler/src/codegen/components.rs index af5bfd080b..9dc09d8b5e 100644 --- a/crates/compiler/src/codegen/components.rs +++ b/crates/compiler/src/codegen/components.rs @@ -19,7 +19,9 @@ pub const VERSION_CUSTOM_SECTION: &str = ".rune_version"; pub const RESOURCE_CUSTOM_SECTION: &str = ".rune_resource"; /// A file that will be written to the Rune's build directory. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct File { pub path: PathBuf, pub data: Arc<[u8]>, @@ -72,7 +74,9 @@ impl CustomSection { } } -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct RuneVersion { /// The version of the tool generating a Rune, typically what you'd see /// when running `rune --version`. diff --git a/crates/compiler/src/compile/cargo_build.rs b/crates/compiler/src/compile/cargo_build.rs index 67be44ffd3..d6a3814590 100644 --- a/crates/compiler/src/compile/cargo_build.rs +++ b/crates/compiler/src/compile/cargo_build.rs @@ -1,45 +1,21 @@ use std::{ path::Path, process::{Command, Output, Stdio}, - sync::Mutex, }; -use legion::systems::CommandBuffer; - use crate::{ - compile::{CompilationResult, CompileError, CompiledBinary}, - BuildContext, Verbosity, + compile::{CompileError, CompiledBinary}, + Verbosity, }; -#[legion::system] -pub(crate) fn run(cmd: &mut CommandBuffer, #[resource] ctx: &BuildContext) { - let BuildContext { - working_directory, - optimized, - verbosity, - name, - .. - } = ctx; - - rustfmt(working_directory); - - let result = build(name, working_directory, *optimized, *verbosity); - - // Note: the exec_mut() method takes a Fn() closure and not a FnOnce(), so - // we need to use a Mutex> to move the result. - let result = Mutex::new(Some(result)); - cmd.exec_mut(move |_, res| { - let result = result.lock().unwrap().take().unwrap(); - res.insert(CompilationResult(result)); - }) -} - -fn build( +pub fn build( name: &str, working_directory: &Path, optimized: bool, verbosity: Verbosity, ) -> Result { + rustfmt(working_directory); + let mut cmd = Command::new("cargo"); cmd.arg("build") .arg("--manifest-path") diff --git a/crates/compiler/src/compile/components.rs b/crates/compiler/src/compile/components.rs index b3842c9621..e828b50391 100644 --- a/crates/compiler/src/compile/components.rs +++ b/crates/compiler/src/compile/components.rs @@ -26,7 +26,7 @@ impl Deref for CompiledBinary { /// The result from compiling... Essentially a newtype'd `Result`. #[derive(Debug)] -pub struct CompilationResult(pub Result); +pub struct CompilationResult(pub Result>); #[derive(Debug)] pub enum CompileError { diff --git a/crates/compiler/src/compile/mod.rs b/crates/compiler/src/compile/mod.rs index a146676439..1daf7d0f9e 100644 --- a/crates/compiler/src/compile/mod.rs +++ b/crates/compiler/src/compile/mod.rs @@ -2,11 +2,45 @@ mod cargo_build; mod components; mod write_project_to_disk; +use std::sync::Arc; + +use im::Vector; + pub use self::components::*; -use crate::Phase; +use crate::{codegen::File, BuildContext, FeatureFlags}; + +#[salsa::query_group(InputsGroup)] +pub trait Inputs { + #[salsa::input] + fn build_context(&self) -> Arc; + #[salsa::input] + fn feature_flags(&self) -> FeatureFlags; + #[salsa::input] + fn files(&self) -> Vector; +} + +#[salsa::query_group(CompileGroup)] +pub trait Compile: Inputs { + #[salsa::dependencies] + fn build(&self) -> Result>; +} + +fn build(db: &dyn Compile) -> Result> { + let ctx = db.build_context(); + let files = db.files(); + + for file in &files { + write_project_to_disk::run(file, &ctx); + } + + let BuildContext { + name, + working_directory, + optimized, + verbosity, + .. + } = &*ctx; -pub fn phase() -> Phase { - Phase::new() - .and_then(write_project_to_disk::run_system) - .and_then(cargo_build::run_system) + cargo_build::build(name, working_directory, *optimized, *verbosity) + .map_err(Arc::new) } diff --git a/crates/compiler/src/compile/write_project_to_disk.rs b/crates/compiler/src/compile/write_project_to_disk.rs index 9770624562..b9965f79e7 100644 --- a/crates/compiler/src/compile/write_project_to_disk.rs +++ b/crates/compiler/src/compile/write_project_to_disk.rs @@ -1,7 +1,6 @@ use crate::{codegen::File, BuildContext}; -#[legion::system(for_each)] -pub(crate) fn run(File { path, data }: &File, #[resource] ctx: &BuildContext) { +pub(crate) fn run(File { path, data }: &File, ctx: &BuildContext) { let full_path = ctx.working_directory.join(path); if let Some(parent) = full_path.parent() { diff --git a/crates/compiler/src/phases.rs b/crates/compiler/src/phases.rs index 9ec772a42b..9dbf5c236d 100644 --- a/crates/compiler/src/phases.rs +++ b/crates/compiler/src/phases.rs @@ -1,7 +1,8 @@ use legion::{systems::Runnable, Resources, World}; use crate::{ - codegen, compile, + codegen, + compile::{CompilationResult, Compile}, hooks::{Continuation, Ctx, Hooks}, lowering, parse, type_check, BuildContext, FeatureFlags, }; @@ -21,6 +22,8 @@ pub fn build_with_hooks( features: FeatureFlags, hooks: &mut dyn Hooks, ) -> (World, Resources) { + let db = Database::default(); + let mut world = World::default(); let mut res = Resources::default(); @@ -68,7 +71,8 @@ pub fn build_with_hooks( return (world, res); } - compile::phase().run(&mut world, &mut res); + let result = db.build(); + res.insert(CompilationResult(result)); if hooks.after_compile(&mut c(&mut world, &mut res)) != Continuation::Continue @@ -79,6 +83,14 @@ pub fn build_with_hooks( (world, res) } +#[derive(Default)] +#[salsa::database(crate::compile::CompileGroup, crate::compile::InputsGroup)] +struct Database { + storage: salsa::Storage, +} + +impl salsa::Database for Database {} + /// A group of operations which make up a single "phase" in the build process. pub struct Phase(legion::systems::Builder); From 5953d32258bcbf5c5c2c8674830655c772d4974b Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 8 Apr 2022 00:27:08 +0800 Subject: [PATCH 02/84] Migrated most of codegen to salsa --- crates/compiler/src/codegen/components.rs | 4 +- .../src/codegen/generate_cargo_config.rs | 2 +- .../src/codegen/generate_cargo_toml.rs | 16 ++- .../src/codegen/generate_resource_section.rs | 5 +- .../codegen/generate_rune_graph_section.rs | 5 +- .../src/codegen/generate_version_section.rs | 2 +- crates/compiler/src/codegen/mod.rs | 108 +++++++++++++++++- crates/compiler/src/compile/mod.rs | 16 +-- crates/compiler/src/inputs.rs | 70 ++++++++++++ crates/compiler/src/lib.rs | 1 + crates/compiler/src/lowering/components.rs | 27 ++++- crates/compiler/src/phases.rs | 61 +++++++++- 12 files changed, 284 insertions(+), 33 deletions(-) create mode 100644 crates/compiler/src/inputs.rs diff --git a/crates/compiler/src/codegen/components.rs b/crates/compiler/src/codegen/components.rs index 9dc09d8b5e..4d5756831a 100644 --- a/crates/compiler/src/codegen/components.rs +++ b/crates/compiler/src/codegen/components.rs @@ -37,7 +37,9 @@ impl File { } /// A WebAssembly custom section to be embedded in the Rune. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct CustomSection { pub section_name: String, pub value: Arc<[u8]>, diff --git a/crates/compiler/src/codegen/generate_cargo_config.rs b/crates/compiler/src/codegen/generate_cargo_config.rs index 79aff8553b..c5f39160fa 100644 --- a/crates/compiler/src/codegen/generate_cargo_config.rs +++ b/crates/compiler/src/codegen/generate_cargo_config.rs @@ -9,7 +9,7 @@ pub(crate) fn run(cmd: &mut CommandBuffer, #[resource] ctx: &BuildContext) { cmd.push((config,)); } -fn generate_config(optimized: bool) -> File { +pub(crate) fn generate_config(optimized: bool) -> File { let target = if optimized { Some(Targets { wasm32_unknown_unknown: Target { diff --git a/crates/compiler/src/codegen/generate_cargo_toml.rs b/crates/compiler/src/codegen/generate_cargo_toml.rs index 7dc3894477..fb764135c4 100644 --- a/crates/compiler/src/codegen/generate_cargo_toml.rs +++ b/crates/compiler/src/codegen/generate_cargo_toml.rs @@ -21,6 +21,17 @@ pub(crate) fn run( #[resource] features: &FeatureFlags, query: &mut Query<&ProcBlock>, ) { + let proc_blocks: Vec<_> = query.iter(world).collect(); + + let file = generate(features, proc_blocks.iter().map(|&p| p), ctx); + cmd.push((file,)); +} + +pub(crate) fn generate<'a>( + features: &FeatureFlags, + proc_blocks: impl Iterator + 'a, + ctx: &BuildContext, +) -> File { let core_version = hotg_rune_core::VERSION; if core_version.contains("-dev") && features.rune_repo_dir.is_none() { @@ -43,7 +54,6 @@ pub(crate) fn run( ); } - let proc_blocks = query.iter(world); let mut manifest = generate_manifest(proc_blocks, &ctx.name, &ctx.current_directory); @@ -53,8 +63,8 @@ pub(crate) fn run( let manifest = toml::to_string_pretty(&manifest) .expect("Serializing to a string should never fail"); - let file = File::new("Cargo.toml", manifest.into_bytes()); - cmd.push((file,)); + + File::new("Cargo.toml", manifest.into_bytes()) } // Generate the `Cargo.toml` manifest. diff --git a/crates/compiler/src/codegen/generate_resource_section.rs b/crates/compiler/src/codegen/generate_resource_section.rs index 90dcd1aee8..9e2469e44e 100644 --- a/crates/compiler/src/codegen/generate_resource_section.rs +++ b/crates/compiler/src/codegen/generate_resource_section.rs @@ -20,7 +20,10 @@ pub(crate) fn run( }); } -fn inline_resource(name: &Name, data: &ResourceData) -> CustomSection { +pub(crate) fn inline_resource( + name: &Name, + data: &ResourceData, +) -> CustomSection { let name_len = u32::try_from(name.len()).unwrap(); let data_len = u32::try_from(data.len()).unwrap(); let buffer_length = std::mem::size_of_val(&name_len) diff --git a/crates/compiler/src/codegen/generate_rune_graph_section.rs b/crates/compiler/src/codegen/generate_rune_graph_section.rs index d568b08639..b1801b6a78 100644 --- a/crates/compiler/src/codegen/generate_rune_graph_section.rs +++ b/crates/compiler/src/codegen/generate_rune_graph_section.rs @@ -11,7 +11,8 @@ use legion::{ use super::{CapabilitySummary, RuneSummary}; use crate::{ codegen::{ - ModelSummary, OutputSummary, ProcBlockSummary, RuneGraph, TensorId, + Codegen, CustomSection, ModelSummary, OutputSummary, ProcBlockSummary, + RuneGraph, TensorId, }, lowering::{ self, Inputs, Model, ModelFile, Name, Outputs, ProcBlock, Resource, @@ -21,6 +22,8 @@ use crate::{ BuildContext, }; +pub(crate) fn rune_graph_section(_db: &dyn Codegen) -> CustomSection { todo!() } + /// Generate an abbreviated [`RuneGraph`]. #[legion::system] pub(crate) fn run( diff --git a/crates/compiler/src/codegen/generate_version_section.rs b/crates/compiler/src/codegen/generate_version_section.rs index afd85cc31a..94be072faa 100644 --- a/crates/compiler/src/codegen/generate_version_section.rs +++ b/crates/compiler/src/codegen/generate_version_section.rs @@ -10,7 +10,7 @@ pub(crate) fn run(cmd: &mut CommandBuffer, #[resource] ctx: &BuildContext) { } } -fn version_section(ctx: &BuildContext) -> Option { +pub(crate) fn version_section(ctx: &BuildContext) -> Option { ctx.rune_version.as_ref().map(|version| { version .as_custom_section() diff --git a/crates/compiler/src/codegen/mod.rs b/crates/compiler/src/codegen/mod.rs index f99ac40a02..9d3ec26aa5 100644 --- a/crates/compiler/src/codegen/mod.rs +++ b/crates/compiler/src/codegen/mod.rs @@ -14,10 +14,19 @@ mod generate_rune_graph_section; mod generate_rust_toolchain_toml; mod generate_version_section; +use std::{path::Path, sync::Arc}; + pub use components::*; +use im::Vector; use legion::Registry; -use crate::{phases::Phase, serialize::RegistryExt}; +use crate::{ + codegen::generate_rune_graph_section::rune_graph_section, + inputs::Inputs, + lowering::{Name, ResourceData}, + phases::Phase, + serialize::RegistryExt, +}; pub fn phase() -> Phase { Phase::new() @@ -39,3 +48,100 @@ pub(crate) fn register_components(registry: &mut Registry) { .register_with_type_name::() .register_with_type_name::(); } + +#[salsa::query_group(CodegenGroup)] +pub trait Codegen: Inputs { + fn rust_toolchain_toml(&self) -> File; + fn cargo_config(&self) -> File; + fn cargo_toml(&self) -> File; + fn model_files(&self) -> Vector; + + fn resource_section(&self, name: Name, data: ResourceData) + -> CustomSection; + fn resource_sections(&self) -> Vector; + fn version_section(&self) -> Option; + fn rune_graph_section(&self) -> CustomSection; + fn lib_rs(&self) -> File; + + fn custom_sections(&self) -> Vector; + + fn files(&self) -> Vector; +} + +fn rust_toolchain_toml(_: &dyn Codegen) -> File { + let rust_toolchain = crate::rust_toolchain(); + let contents = toml::to_vec(&rust_toolchain) + .expect("We can always serialize a hard-coded TOML object"); + + File::new("rust-toolchain.toml", contents) +} + +fn cargo_config(db: &dyn Codegen) -> File { + let ctx = db.build_context(); + generate_cargo_config::generate_config(ctx.optimized) +} + +fn cargo_toml(db: &dyn Codegen) -> File { + let features = db.feature_flags(); + let proc_blocks = db.all_proc_blocks(); + let ctx = db.build_context(); + + generate_cargo_toml::generate(&features, proc_blocks.iter(), &ctx) +} + +fn model_files(db: &dyn Codegen) -> Vector { + let mut files = Vector::new(); + + for (name, data) in db.all_model_data() { + let path = Path::new("models").join(name.as_str()); + let file = File::new(path, Arc::clone(&data.0)); + files.push_back(file); + } + + files +} + +fn resource_section( + _: &dyn Codegen, + name: Name, + data: ResourceData, +) -> CustomSection { + generate_resource_section::inline_resource(&name, &data) +} + +fn resource_sections(db: &dyn Codegen) -> Vector { + db.all_resource_data() + .into_iter() + .map(|(name, data)| db.resource_section(name, data)) + .collect() +} + +fn version_section(db: &dyn Codegen) -> Option { + let ctx = db.build_context(); + generate_version_section::version_section(&ctx) +} + +fn lib_rs(db: &dyn Codegen) -> File { todo!() } + +fn custom_sections(db: &dyn Codegen) -> Vector { + let mut sections = Vector::new(); + + sections.push_back(db.rune_graph_section()); + if let Some(version) = db.version_section() { + sections.push_back(version); + } + sections.extend(db.resource_sections()); + + sections +} + +fn files(db: &dyn Codegen) -> Vector { + let mut files = Vector::new(); + + files.push_back(db.rust_toolchain_toml()); + files.push_back(db.cargo_config()); + files.push_back(db.cargo_toml()); + files.extend(db.model_files()); + + files +} diff --git a/crates/compiler/src/compile/mod.rs b/crates/compiler/src/compile/mod.rs index 1daf7d0f9e..f3f0dfe793 100644 --- a/crates/compiler/src/compile/mod.rs +++ b/crates/compiler/src/compile/mod.rs @@ -4,23 +4,11 @@ mod write_project_to_disk; use std::sync::Arc; -use im::Vector; - pub use self::components::*; -use crate::{codegen::File, BuildContext, FeatureFlags}; - -#[salsa::query_group(InputsGroup)] -pub trait Inputs { - #[salsa::input] - fn build_context(&self) -> Arc; - #[salsa::input] - fn feature_flags(&self) -> FeatureFlags; - #[salsa::input] - fn files(&self) -> Vector; -} +use crate::{inputs::Inputs, BuildContext, codegen::Codegen}; #[salsa::query_group(CompileGroup)] -pub trait Compile: Inputs { +pub trait Compile: Inputs + Codegen { #[salsa::dependencies] fn build(&self) -> Result>; } diff --git a/crates/compiler/src/inputs.rs b/crates/compiler/src/inputs.rs new file mode 100644 index 0000000000..2214842d93 --- /dev/null +++ b/crates/compiler/src/inputs.rs @@ -0,0 +1,70 @@ +use std::{path::Path, sync::Arc}; + +use im::Vector; + +use crate::{ + lowering::{Model, ModelData, Name, ProcBlock, Resource, ResourceData}, + BuildContext, FeatureFlags, +}; + +pub trait FileSystem { + fn read_file(&self, path: &Path) -> Result, std::io::Error> { + std::fs::read(path).map(Vector::from) + } +} + +#[salsa::query_group(InputsGroup)] +pub trait Inputs: FileSystem { + #[salsa::input] + fn build_context(&self) -> Arc; + #[salsa::input] + fn feature_flags(&self) -> FeatureFlags; + + #[salsa::input] + fn node_inputs(&self, name: Name) -> crate::lowering::Inputs; + #[salsa::input] + fn node_outputs(&self, name: Name) -> crate::lowering::Outputs; + + #[salsa::input] + fn resource_names(&self) -> Vector; + #[salsa::input] + fn resource_info(&self, name: Name) -> Vector; + #[salsa::input] + fn resource_data(&self, name: Name) -> ResourceData; + fn all_resource_data(&self) -> Vector<(Name, ResourceData)>; + + #[salsa::input] + fn proc_block_names(&self) -> Vector; + #[salsa::input] + fn proc_block_info(&self, name: Name) -> ProcBlock; + fn all_proc_blocks(&self) -> Vector; + + #[salsa::input] + fn model_names(&self) -> Vector; + #[salsa::input] + fn model_info(&self, name: Name) -> Model; + #[salsa::input] + fn model_data(&self, name: Name) -> ModelData; + fn all_model_data(&self) -> Vector<(Name, ModelData)>; +} + +fn all_model_data(db: &dyn Inputs) -> Vector<(Name, ModelData)> { + db.model_names() + .into_iter() + .map(|name| (name.clone(), db.model_data(name))) + .collect() +} + +fn all_resource_data(db: &dyn Inputs) -> Vector<(Name, ResourceData)> { + db.resource_names() + .into_iter() + .map(|name| (name.clone(), db.resource_data(name))) + .collect() +} + +fn all_proc_blocks(db: &dyn Inputs) -> Vector { + db.proc_block_names() + .into_iter() + .map(|name| db.proc_block_info(name)) + .collect() +} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index e9866b8abb..dfa909c42a 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -40,6 +40,7 @@ mod phases; pub mod serialize; mod toolchain; pub mod type_check; +mod inputs; pub use crate::{ build_context::{BuildContext, FeatureFlags, Verbosity}, diff --git a/crates/compiler/src/lowering/components.rs b/crates/compiler/src/lowering/components.rs index ca5a5f2e1f..1edf1633df 100644 --- a/crates/compiler/src/lowering/components.rs +++ b/crates/compiler/src/lowering/components.rs @@ -135,7 +135,7 @@ impl Display for SourceKind { } } -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub struct ProcBlock { pub path: Path, @@ -152,6 +152,14 @@ impl ProcBlock { } } +// TODO: remove this +impl Hash for ProcBlock { + fn hash(&self, state: &mut H) { + self.path.hash(state); + self.parameters.iter().for_each(|pair| pair.hash(state)); + } +} + #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Resource { /// Where to read the [`Resource`]'s default value from. @@ -265,13 +273,22 @@ pub struct Outputs { /// The list of [`Tensor`]s that may be the inputs to a [`PipelineNode`]. #[derive( - Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize, + Debug, + Default, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, )] pub struct Inputs { pub tensors: Vec, } -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct ResourceData(pub Arc<[u8]>); impl>> From for ResourceData { @@ -288,7 +305,9 @@ impl Deref for ResourceData { fn deref(&self) -> &Self::Target { &self.0 } } -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct ModelData(pub Arc<[u8]>); impl>> From for ModelData { diff --git a/crates/compiler/src/phases.rs b/crates/compiler/src/phases.rs index 9dbf5c236d..0ffc92d96d 100644 --- a/crates/compiler/src/phases.rs +++ b/crates/compiler/src/phases.rs @@ -1,10 +1,15 @@ -use legion::{systems::Runnable, Resources, World}; +use std::sync::Arc; + +use im::Vector; +use legion::{systems::Runnable, IntoQuery, Resources, World}; use crate::{ - codegen, + codegen::{self, Codegen}, compile::{CompilationResult, Compile}, hooks::{Continuation, Ctx, Hooks}, - lowering, parse, type_check, BuildContext, FeatureFlags, + inputs::Inputs, + lowering::{self, Model, Name, Outputs, ProcBlock}, + parse, type_check, BuildContext, FeatureFlags, }; /// Execute the `rune build` process. @@ -22,7 +27,9 @@ pub fn build_with_hooks( features: FeatureFlags, hooks: &mut dyn Hooks, ) -> (World, Resources) { - let db = Database::default(); + let mut db = Database::default(); + db.set_build_context(Arc::new(ctx.clone())); + db.set_feature_flags(features.clone()); let mut world = World::default(); let mut res = Resources::default(); @@ -63,7 +70,10 @@ pub fn build_with_hooks( } log::debug!("Beginning the \"codegen\" phase"); - codegen::phase().run(&mut world, &mut res); + + update_db_before_codegen(&world, &mut db); + + let _files = db.files(); if hooks.after_codegen(&mut c(&mut world, &mut res)) != Continuation::Continue @@ -83,13 +93,52 @@ pub fn build_with_hooks( (world, res) } +fn update_db_before_codegen(world: &World, db: &mut Database) { + let mut pb_names = Vector::new(); + <( + &Name, + &crate::lowering::ProcBlock, + &crate::lowering::Inputs, + &crate::lowering::Outputs, + )>::query() + .for_each(world, |(n, p, i, o)| { + pb_names.push_back(n.clone()); + db.set_node_inputs(n.clone(), i.clone()); + db.set_node_outputs(n.clone(), o.clone()); + db.set_proc_block_info(n.clone(), p.clone()); + }); + db.set_proc_block_names(pb_names); + + let mut model_names = Vector::new(); + <( + &Name, + &crate::lowering::Model, + &crate::lowering::ModelData, + &crate::lowering::Inputs, + &crate::lowering::Outputs, + )>::query() + .for_each(world, |(n, m, d, i, o)| { + model_names.push_back(n.clone()); + db.set_node_inputs(n.clone(), i.clone()); + db.set_node_outputs(n.clone(), o.clone()); + db.set_model_info(n.clone(), m.clone()); + db.set_model_data(n.clone(), d.clone()); + }); + db.set_model_names(model_names); +} + #[derive(Default)] -#[salsa::database(crate::compile::CompileGroup, crate::compile::InputsGroup)] +#[salsa::database( + crate::codegen::CodegenGroup, + crate::compile::CompileGroup, + crate::inputs::InputsGroup +)] struct Database { storage: salsa::Storage, } impl salsa::Database for Database {} +impl crate::inputs::FileSystem for Database {} /// A group of operations which make up a single "phase" in the build process. pub struct Phase(legion::systems::Builder); From 0bd4eb2f77edae420354b9f1385edb57a3a129f7 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 8 Apr 2022 00:46:55 +0800 Subject: [PATCH 03/84] Migrated the parse phase over to salsa --- crates/compiler/src/codegen/inputs.rs | 57 +++++++++++++++++++++++++++ crates/compiler/src/codegen/mod.rs | 10 +++-- crates/compiler/src/inputs.rs | 53 +------------------------ crates/compiler/src/parse/mod.rs | 16 +++++++- crates/compiler/src/phases.rs | 19 +++++++-- 5 files changed, 94 insertions(+), 61 deletions(-) create mode 100644 crates/compiler/src/codegen/inputs.rs diff --git a/crates/compiler/src/codegen/inputs.rs b/crates/compiler/src/codegen/inputs.rs new file mode 100644 index 0000000000..c27af7559b --- /dev/null +++ b/crates/compiler/src/codegen/inputs.rs @@ -0,0 +1,57 @@ +use im::Vector; + +use crate::{ + inputs::Inputs, + lowering::{Model, ModelData, Name, ProcBlock, Resource, ResourceData}, +}; + +#[salsa::query_group(CodegenInputsGroup)] +pub trait CodegenInputs: Inputs { + #[salsa::input] + fn node_inputs(&self, name: Name) -> crate::lowering::Inputs; + #[salsa::input] + fn node_outputs(&self, name: Name) -> crate::lowering::Outputs; + + #[salsa::input] + fn resource_names(&self) -> Vector; + #[salsa::input] + fn resource_info(&self, name: Name) -> Vector; + #[salsa::input] + fn resource_data(&self, name: Name) -> ResourceData; + fn all_resource_data(&self) -> Vector<(Name, ResourceData)>; + + #[salsa::input] + fn proc_block_names(&self) -> Vector; + #[salsa::input] + fn proc_block_info(&self, name: Name) -> ProcBlock; + fn all_proc_blocks(&self) -> Vector; + + #[salsa::input] + fn model_names(&self) -> Vector; + #[salsa::input] + fn model_info(&self, name: Name) -> Model; + #[salsa::input] + fn model_data(&self, name: Name) -> ModelData; + fn all_model_data(&self) -> Vector<(Name, ModelData)>; +} + +fn all_model_data(db: &dyn CodegenInputs) -> Vector<(Name, ModelData)> { + db.model_names() + .into_iter() + .map(|name| (name.clone(), db.model_data(name))) + .collect() +} + +fn all_resource_data(db: &dyn CodegenInputs) -> Vector<(Name, ResourceData)> { + db.resource_names() + .into_iter() + .map(|name| (name.clone(), db.resource_data(name))) + .collect() +} + +fn all_proc_blocks(db: &dyn CodegenInputs) -> Vector { + db.proc_block_names() + .into_iter() + .map(|name| db.proc_block_info(name)) + .collect() +} diff --git a/crates/compiler/src/codegen/mod.rs b/crates/compiler/src/codegen/mod.rs index 9d3ec26aa5..8189e0255d 100644 --- a/crates/compiler/src/codegen/mod.rs +++ b/crates/compiler/src/codegen/mod.rs @@ -13,6 +13,7 @@ mod generate_resource_section; mod generate_rune_graph_section; mod generate_rust_toolchain_toml; mod generate_version_section; +pub(crate) mod inputs; use std::{path::Path, sync::Arc}; @@ -21,8 +22,9 @@ use im::Vector; use legion::Registry; use crate::{ - codegen::generate_rune_graph_section::rune_graph_section, - inputs::Inputs, + codegen::{ + generate_rune_graph_section::rune_graph_section, inputs::CodegenInputs, + }, lowering::{Name, ResourceData}, phases::Phase, serialize::RegistryExt, @@ -50,7 +52,7 @@ pub(crate) fn register_components(registry: &mut Registry) { } #[salsa::query_group(CodegenGroup)] -pub trait Codegen: Inputs { +pub trait Codegen: CodegenInputs { fn rust_toolchain_toml(&self) -> File; fn cargo_config(&self) -> File; fn cargo_toml(&self) -> File; @@ -121,7 +123,7 @@ fn version_section(db: &dyn Codegen) -> Option { generate_version_section::version_section(&ctx) } -fn lib_rs(db: &dyn Codegen) -> File { todo!() } +fn lib_rs(_db: &dyn Codegen) -> File { todo!() } fn custom_sections(db: &dyn Codegen) -> Vector { let mut sections = Vector::new(); diff --git a/crates/compiler/src/inputs.rs b/crates/compiler/src/inputs.rs index 2214842d93..63a1ff03f3 100644 --- a/crates/compiler/src/inputs.rs +++ b/crates/compiler/src/inputs.rs @@ -2,10 +2,7 @@ use std::{path::Path, sync::Arc}; use im::Vector; -use crate::{ - lowering::{Model, ModelData, Name, ProcBlock, Resource, ResourceData}, - BuildContext, FeatureFlags, -}; +use crate::{BuildContext, FeatureFlags}; pub trait FileSystem { fn read_file(&self, path: &Path) -> Result, std::io::Error> { @@ -19,52 +16,4 @@ pub trait Inputs: FileSystem { fn build_context(&self) -> Arc; #[salsa::input] fn feature_flags(&self) -> FeatureFlags; - - #[salsa::input] - fn node_inputs(&self, name: Name) -> crate::lowering::Inputs; - #[salsa::input] - fn node_outputs(&self, name: Name) -> crate::lowering::Outputs; - - #[salsa::input] - fn resource_names(&self) -> Vector; - #[salsa::input] - fn resource_info(&self, name: Name) -> Vector; - #[salsa::input] - fn resource_data(&self, name: Name) -> ResourceData; - fn all_resource_data(&self) -> Vector<(Name, ResourceData)>; - - #[salsa::input] - fn proc_block_names(&self) -> Vector; - #[salsa::input] - fn proc_block_info(&self, name: Name) -> ProcBlock; - fn all_proc_blocks(&self) -> Vector; - - #[salsa::input] - fn model_names(&self) -> Vector; - #[salsa::input] - fn model_info(&self, name: Name) -> Model; - #[salsa::input] - fn model_data(&self, name: Name) -> ModelData; - fn all_model_data(&self) -> Vector<(Name, ModelData)>; -} - -fn all_model_data(db: &dyn Inputs) -> Vector<(Name, ModelData)> { - db.model_names() - .into_iter() - .map(|name| (name.clone(), db.model_data(name))) - .collect() -} - -fn all_resource_data(db: &dyn Inputs) -> Vector<(Name, ResourceData)> { - db.resource_names() - .into_iter() - .map(|name| (name.clone(), db.resource_data(name))) - .collect() -} - -fn all_proc_blocks(db: &dyn Inputs) -> Vector { - db.proc_block_names() - .into_iter() - .map(|name| db.proc_block_info(name)) - .collect() } diff --git a/crates/compiler/src/parse/mod.rs b/crates/compiler/src/parse/mod.rs index 7ff8000b27..59ee62463c 100644 --- a/crates/compiler/src/parse/mod.rs +++ b/crates/compiler/src/parse/mod.rs @@ -10,7 +10,10 @@ use codespan_reporting::diagnostic::{Diagnostic, Label}; use legion::{systems::CommandBuffer, Registry}; pub use self::yaml::*; -use crate::{phases::Phase, serialize::RegistryExt, BuildContext, Diagnostics}; +use crate::{ + inputs::Inputs, phases::Phase, serialize::RegistryExt, BuildContext, + Diagnostics, +}; pub fn phase() -> Phase { Phase::with_setup(|res| { @@ -19,6 +22,12 @@ pub fn phase() -> Phase { .and_then(run_system) } +#[salsa::query_group(ParseGroup)] +pub trait Parse: Inputs { + #[salsa::dependencies] + fn parse(&self) -> Result>; +} + #[legion::system] fn run( cmd: &mut CommandBuffer, @@ -39,6 +48,11 @@ fn run( } } +fn parse(db: &dyn Parse) -> Result> { + let ctx = db.build_context(); + Document::parse(&ctx.runefile).map_err(parse_failed_diagnostic) +} + fn parse_failed_diagnostic(e: serde_yaml::Error) -> Diagnostic<()> { let msg = format!("Unable to parse the input: {}", e); diff --git a/crates/compiler/src/phases.rs b/crates/compiler/src/phases.rs index 0ffc92d96d..58c4446a36 100644 --- a/crates/compiler/src/phases.rs +++ b/crates/compiler/src/phases.rs @@ -4,12 +4,13 @@ use im::Vector; use legion::{systems::Runnable, IntoQuery, Resources, World}; use crate::{ - codegen::{self, Codegen}, + codegen::{inputs::CodegenInputs, Codegen}, compile::{CompilationResult, Compile}, hooks::{Continuation, Ctx, Hooks}, inputs::Inputs, - lowering::{self, Model, Name, Outputs, ProcBlock}, - parse, type_check, BuildContext, FeatureFlags, + lowering::{self, Name}, + parse::Parse, + type_check, BuildContext, Diagnostics, FeatureFlags, }; /// Execute the `rune build` process. @@ -36,6 +37,7 @@ pub fn build_with_hooks( res.insert(ctx); res.insert(features); + res.insert(Diagnostics::default()); if hooks.before_parse(&mut c(&mut world, &mut res)) != Continuation::Continue @@ -44,7 +46,14 @@ pub fn build_with_hooks( } log::debug!("Beginning the \"parse\" phase"); - parse::phase().run(&mut world, &mut res); + match db.parse() { + Ok(d) => { + res.insert(d.clone().to_v1()); + }, + Err(e) => { + res.get_mut_or_default::().push(e); + }, + } if hooks.after_parse(&mut c(&mut world, &mut res)) != Continuation::Continue { @@ -129,6 +138,8 @@ fn update_db_before_codegen(world: &World, db: &mut Database) { #[derive(Default)] #[salsa::database( + crate::parse::ParseGroup, + crate::codegen::inputs::CodegenInputsGroup, crate::codegen::CodegenGroup, crate::compile::CompileGroup, crate::inputs::InputsGroup From 562eacf07da835edfe514bfcf9824df06a1ccfad Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sun, 17 Apr 2022 18:30:38 +0800 Subject: [PATCH 04/84] Use the bytes crate for representing bytes --- Cargo.lock | 10 ++++++++++ crates/compiler/Cargo.toml | 1 + crates/compiler/src/codegen/components.rs | 10 +++++----- crates/compiler/src/codegen/generate_model_files.rs | 4 ++-- crates/compiler/src/codegen/mod.rs | 4 ++-- crates/compiler/src/compile/components.rs | 6 ++++-- crates/compiler/src/inputs.rs | 6 +++--- crates/compiler/src/lowering/components.rs | 10 +++++----- crates/compiler/src/lowering/load_resource_data.rs | 2 +- 9 files changed, 33 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fcdcd9bbc1..9d5f0db040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -304,6 +304,15 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +dependencies = [ + "serde", +] + [[package]] name = "bzip2" version = "0.4.3" @@ -1141,6 +1150,7 @@ name = "hotg-rune-compiler" version = "0.11.3" dependencies = [ "atomic_refcell", + "bytes", "cargo_toml", "codespan", "codespan-reporting", diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index 3d1b400761..f86c047976 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -15,6 +15,7 @@ readme = "README.md" [dependencies] atomic_refcell = "0.1.8" +bytes = { version = "1.1.0", features = ["serde"] } cargo_toml = "0.10.3" codespan = { version = "0.11.1", features = ["serialization"] } codespan-reporting = "0.11.1" diff --git a/crates/compiler/src/codegen/components.rs b/crates/compiler/src/codegen/components.rs index 4d5756831a..4b3065faa0 100644 --- a/crates/compiler/src/codegen/components.rs +++ b/crates/compiler/src/codegen/components.rs @@ -3,9 +3,9 @@ use std::{ fmt::{self, Display, Formatter}, ops::Deref, path::PathBuf, - sync::Arc, }; +use bytes::Bytes; use hotg_rune_core::Shape; use serde::Serialize; @@ -24,11 +24,11 @@ pub const RESOURCE_CUSTOM_SECTION: &str = ".rune_resource"; )] pub struct File { pub path: PathBuf, - pub data: Arc<[u8]>, + pub data: Bytes, } impl File { - pub fn new(path: impl Into, data: impl Into>) -> Self { + pub fn new(path: impl Into, data: impl Into) -> Self { File { path: path.into(), data: data.into(), @@ -42,11 +42,11 @@ impl File { )] pub struct CustomSection { pub section_name: String, - pub value: Arc<[u8]>, + pub value: Bytes, } impl CustomSection { - pub fn new(name: impl Into, value: impl Into>) -> Self { + pub fn new(name: impl Into, value: impl Into) -> Self { let section_name = name.into(); let value = value.into(); diff --git a/crates/compiler/src/codegen/generate_model_files.rs b/crates/compiler/src/codegen/generate_model_files.rs index 52ae03d0ba..ac3a608ddf 100644 --- a/crates/compiler/src/codegen/generate_model_files.rs +++ b/crates/compiler/src/codegen/generate_model_files.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::path::Path; use legion::systems::CommandBuffer; @@ -12,6 +12,6 @@ use crate::{ #[legion::system(for_each)] pub(crate) fn run(cmd: &mut CommandBuffer, name: &Name, data: &ModelData) { let path = Path::new("models").join(name.as_str()); - let file = File::new(path, Arc::clone(&data.0)); + let file = File::new(path, data.0.clone()); cmd.push((file,)); } diff --git a/crates/compiler/src/codegen/mod.rs b/crates/compiler/src/codegen/mod.rs index 8189e0255d..22ebf9a12e 100644 --- a/crates/compiler/src/codegen/mod.rs +++ b/crates/compiler/src/codegen/mod.rs @@ -15,7 +15,7 @@ mod generate_rust_toolchain_toml; mod generate_version_section; pub(crate) mod inputs; -use std::{path::Path, sync::Arc}; +use std::path::Path; pub use components::*; use im::Vector; @@ -96,7 +96,7 @@ fn model_files(db: &dyn Codegen) -> Vector { for (name, data) in db.all_model_data() { let path = Path::new("models").join(name.as_str()); - let file = File::new(path, Arc::clone(&data.0)); + let file = File::new(path, data.0.clone()); files.push_back(file); } diff --git a/crates/compiler/src/compile/components.rs b/crates/compiler/src/compile/components.rs index e828b50391..04da6d3b33 100644 --- a/crates/compiler/src/compile/components.rs +++ b/crates/compiler/src/compile/components.rs @@ -7,8 +7,10 @@ use std::{ sync::Arc, }; +use bytes::Bytes; + #[derive(Debug, Clone, PartialEq)] -pub struct CompiledBinary(pub Arc<[u8]>); +pub struct CompiledBinary(pub Bytes); impl From> for CompiledBinary { fn from(bytes: Vec) -> Self { CompiledBinary(bytes.into()) } @@ -19,7 +21,7 @@ impl AsRef<[u8]> for CompiledBinary { } impl Deref for CompiledBinary { - type Target = Arc<[u8]>; + type Target = Bytes; fn deref(&self) -> &Self::Target { &self.0 } } diff --git a/crates/compiler/src/inputs.rs b/crates/compiler/src/inputs.rs index 63a1ff03f3..cf076f09e0 100644 --- a/crates/compiler/src/inputs.rs +++ b/crates/compiler/src/inputs.rs @@ -1,12 +1,12 @@ use std::{path::Path, sync::Arc}; -use im::Vector; +use bytes::Bytes; use crate::{BuildContext, FeatureFlags}; pub trait FileSystem { - fn read_file(&self, path: &Path) -> Result, std::io::Error> { - std::fs::read(path).map(Vector::from) + fn read_file(&self, path: &Path) -> Result { + std::fs::read(path).map(Bytes::from) } } diff --git a/crates/compiler/src/lowering/components.rs b/crates/compiler/src/lowering/components.rs index 1edf1633df..4b5dc4f7f1 100644 --- a/crates/compiler/src/lowering/components.rs +++ b/crates/compiler/src/lowering/components.rs @@ -6,9 +6,9 @@ use std::{ hash::Hash, ops::Deref, path::PathBuf, - sync::Arc, }; +use bytes::Bytes; use hotg_rune_core::Shape; use indexmap::IndexMap; use legion::Entity; @@ -289,9 +289,9 @@ pub struct Inputs { #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] -pub struct ResourceData(pub Arc<[u8]>); +pub struct ResourceData(pub Bytes); -impl>> From for ResourceData { +impl> From for ResourceData { fn from(data: T) -> Self { ResourceData(data.into()) } } @@ -308,9 +308,9 @@ impl Deref for ResourceData { #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] -pub struct ModelData(pub Arc<[u8]>); +pub struct ModelData(pub Bytes); -impl>> From for ModelData { +impl> From for ModelData { fn from(data: A) -> Self { ModelData(data.into()) } } diff --git a/crates/compiler/src/lowering/load_resource_data.rs b/crates/compiler/src/lowering/load_resource_data.rs index 1936cc1695..031306c417 100644 --- a/crates/compiler/src/lowering/load_resource_data.rs +++ b/crates/compiler/src/lowering/load_resource_data.rs @@ -34,7 +34,7 @@ pub(crate) fn run( } }, Some(ResourceSource::Inline(data)) => { - cmd.add_component(entity, ResourceData::from(data.as_bytes())); + cmd.add_component(entity, ResourceData::from(data.clone())); }, None => {}, } From a1704ac0dccdcfdd533c2e1e5b2b49c9f7d9dceb Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 18 Apr 2022 22:47:38 +0800 Subject: [PATCH 05/84] Started adapting the lowering code --- Cargo.lock | 30 ++- crates/compiler/Cargo.toml | 4 +- crates/compiler/error-codes.md | 9 + crates/compiler/src/lowering/components.rs | 8 +- crates/compiler/src/lowering/mod.rs | 3 +- crates/compiler/src/lowering/query.rs | 232 ++++++++++++++++++ .../src/lowering/register_resources.rs | 2 +- .../compiler/src/lowering/register_stages.rs | 2 +- .../compiler/src/lowering/register_tensors.rs | 2 +- crates/compiler/src/parse/mod.rs | 80 ++---- crates/compiler/src/parse/yaml.rs | 4 +- crates/compiler/src/phases.rs | 8 +- crates/compiler/src/serialize.rs | 1 - 13 files changed, 310 insertions(+), 75 deletions(-) create mode 100644 crates/compiler/error-codes.md create mode 100644 crates/compiler/src/lowering/query.rs diff --git a/Cargo.lock b/Cargo.lock index 9d5f0db040..d913d583c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1164,6 +1164,7 @@ dependencies = [ "jsonschema", "legion", "log", + "miette", "once_cell", "pretty_assertions 1.2.0", "proc-macro2", @@ -1176,6 +1177,7 @@ dependencies = [ "serde_yaml", "thiserror", "toml", + "tracing", "zip", ] @@ -1324,6 +1326,7 @@ dependencies = [ "bitmaps", "rand_core 0.5.1", "rand_xoshiro", + "serde", "sized-chunks", "typenum", "version_check", @@ -1657,6 +1660,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miette" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ff8e2f24552a78cbf93f2b197f6a788192200c7200648514647cce438714bf" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7119608a3ee194199d537849bb7b600f9e04bb1c91b98fb2620b9a867f17ef20" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2882,9 +2908,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if", "log", diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index f86c047976..a2660b0ef3 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -22,11 +22,12 @@ codespan-reporting = "0.11.1" heck = "0.4.0" hotg-rune-core = { path = "../rune-core", version = "^0.11.0"} hotg-rune-proc-blocks = { path = "../proc-blocks", version = "^0.11.0", default-features = false } -im = "15.0.0" +im = { version = "15.0.0", features = ["serde"] } indexmap = { version = "1.8.0", features = ["serde-1"] } indoc = "1.0.3" legion = { version = "0.4.0", default-features = false, features = ["serialize", "codegen", "extended-tuple-impls"] } log = "0.4.14" +miette = "4.4.0" once_cell = "1.9.0" proc-macro2 = "1.0.36" quote = "1.0.14" @@ -38,6 +39,7 @@ serde_json = "1.0.74" serde_yaml = "0.8.23" thiserror = "1.0.30" toml = "0.5.8" +tracing = { version = "0.1.34", features = ["attributes"] } zip = "0.5.13" [dev-dependencies] diff --git a/crates/compiler/error-codes.md b/crates/compiler/error-codes.md new file mode 100644 index 0000000000..e116a71a45 --- /dev/null +++ b/crates/compiler/error-codes.md @@ -0,0 +1,9 @@ +# Error Codes + + +## E001: Parse Failed + + + +## E002: A resource can't specify both "path" and "inline" values + diff --git a/crates/compiler/src/lowering/components.rs b/crates/compiler/src/lowering/components.rs index 4b5dc4f7f1..1d0406dfde 100644 --- a/crates/compiler/src/lowering/components.rs +++ b/crates/compiler/src/lowering/components.rs @@ -160,14 +160,18 @@ impl Hash for ProcBlock { } } -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub struct Resource { /// Where to read the [`Resource`]'s default value from. pub default_value: Option, pub ty: ResourceType, } -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] pub enum ResourceSource { /// The value is specified in-line as a string. Inline(String), diff --git a/crates/compiler/src/lowering/mod.rs b/crates/compiler/src/lowering/mod.rs index 60596a3065..4ced7ba71b 100644 --- a/crates/compiler/src/lowering/mod.rs +++ b/crates/compiler/src/lowering/mod.rs @@ -3,15 +3,16 @@ mod components; mod load_model_data; mod load_resource_data; +pub mod query; mod register_names; mod register_resources; mod register_stages; mod register_tensors; mod update_nametable; -pub use components::*; use legion::Registry; +pub use self::components::*; use crate::{phases::Phase, serialize::RegistryExt}; pub fn phase() -> Phase { diff --git a/crates/compiler/src/lowering/query.rs b/crates/compiler/src/lowering/query.rs new file mode 100644 index 0000000000..a8527562b1 --- /dev/null +++ b/crates/compiler/src/lowering/query.rs @@ -0,0 +1,232 @@ +use std::{num::NonZeroU32, sync::Arc}; + +use codespan::Span; +use codespan_reporting::diagnostic::{Diagnostic, Label}; +use im::HashMap; + +use crate::{ + inputs::FileSystem, + lowering::{Inputs, Outputs, Resource, ResourceSource}, + parse::{DocumentV1, Image, ResourceDeclaration, ResourceOrString}, + Diagnostics, +}; + +/// This is typically populated by something like [`apply_document()`]. +#[salsa::query_group(LoweringInputsGroup)] +pub trait Ast { + #[salsa::input] + fn image(&self) -> Image; + + #[salsa::input] + fn nodes(&self) -> HashMap, NodeId>; + #[salsa::input] + fn node(&self, id: NodeId) -> Node; + #[salsa::input] + fn node_inputs(&self, id: NodeId) -> Inputs; + #[salsa::input] + fn node_outputs(&self, id: NodeId) -> Outputs; + + #[salsa::input] + fn resources(&self) -> HashMap, ResourceId>; + #[salsa::input] + fn resource(&self, id: ResourceId) -> Resource; +} + +#[salsa::query_group(LoweringGroup)] +pub trait Lowering: Ast + FileSystem {} + +#[must_use = "Diagnostics should always be handled"] +pub fn apply_document(db: &mut dyn Ast, doc: &DocumentV1) -> Diagnostics { + let DocumentV1 { + version: _, + image, + pipeline, + resources, + } = doc; + + let mut diags = Diagnostics::new(); + + db.set_image(image.clone()); + + let mut ids = Counter::new(); + + set_nodes(pipeline, &mut ids, db); + set_resources(resources, &mut ids, db, &mut diags); + + diags +} + +fn set_resources( + resources: &indexmap::IndexMap, + ids: &mut Counter, + db: &mut dyn Ast, + diags: &mut Diagnostics, +) { + let mut res = HashMap::new(); + + for (name, decl) in resources { + let id = ids.resource_id(); + + let source = match (decl.inline.as_ref(), decl.path.as_ref()) { + (Some(inline), None) => { + Some(ResourceSource::Inline(inline.clone())) + }, + (None, Some(path)) => Some(ResourceSource::FromDisk(path.into())), + (None, None) => None, + (Some(_), Some(_)) => { + let diag = PathXorInlineResourceDiagnostic { + name: name.to_string(), + span: decl.span(), + }; + diags.push(diag.into_codespan_diagnostic()); + + continue; + }, + }; + + let resource = Resource { + default_value: source, + ty: decl.ty, + }; + res.insert(Arc::from(name.as_str()), id); + db.set_resource(id, resource); + } + + db.set_resources(res); +} + +#[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)] +#[error("A resource can't specify both \"path\" and \"inline\" values")] +#[diagnostic(code("E002"))] +pub struct PathXorInlineResourceDiagnostic { + pub name: String, + pub span: Span, +} + +impl PathXorInlineResourceDiagnostic { + fn into_codespan_diagnostic(&self) -> Diagnostic<()> { + let msg = format!( + "The resource \"{}\" can't specify both a \"path\" and \"inline\" \ + default value", + self.name + ); + + Diagnostic::error() + .with_message(msg) + .with_labels(vec![Label::primary((), self.span)]) + } +} + +fn set_nodes( + pipeline: &indexmap::IndexMap, + ids: &mut Counter, + db: &mut dyn Ast, +) { + let mut nodes = HashMap::new(); + for (name, stage) in pipeline { + let id = ids.node_id(); + let name: Arc = name.as_str().into(); + nodes.insert(name, id); + db.set_node(id, Node::from_stage(stage)); + } + db.set_nodes(nodes); +} + +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct Node { + kind: NodeKind, + identifier: ResourceOrString, + arguments: HashMap, ResourceOrString>, +} + +impl Node { + fn from_stage(stage: &crate::parse::Stage) -> Self { + let (kind, identifier) = match stage { + crate::parse::Stage::Model(m) => (NodeKind::Model, m.model.clone()), + crate::parse::Stage::ProcBlock(_) => todo!(), + crate::parse::Stage::Capability(_) => todo!(), + crate::parse::Stage::Out(_) => todo!(), + }; + + let arguments = stage + .args() + .iter() + .map(|(k, v)| (Arc::from(k.as_str()), v.0.clone())) + .collect(); + + Node { + kind, + identifier, + arguments, + } + } +} + +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +pub enum NodeKind { + Capability, + Model, + ProcBlock, + Output, +} + +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +#[repr(transparent)] +pub struct NodeId(NonZeroU32); + +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +#[repr(transparent)] +pub struct ResourceId(NonZeroU32); + +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct ArgumentId { + node: NodeId, + name: Arc, +} + +#[derive(Debug)] +struct Counter(u32); + +impl Counter { + fn new() -> Self { Counter(0) } + + fn next(&mut self) -> NonZeroU32 { + self.0 += 1; + NonZeroU32::new(self.0).expect("Unreachable") + } + + fn node_id(&mut self) -> NodeId { NodeId(self.next()) } + + fn resource_id(&mut self) -> ResourceId { ResourceId(self.next()) } +} diff --git a/crates/compiler/src/lowering/register_resources.rs b/crates/compiler/src/lowering/register_resources.rs index 911b012b22..3b7944212c 100644 --- a/crates/compiler/src/lowering/register_resources.rs +++ b/crates/compiler/src/lowering/register_resources.rs @@ -66,7 +66,7 @@ fn path_and_inline_defined_diagnostic( .with_labels(vec![Label::primary((), span)]) } -#[cfg(test)] +#[cfg(never)] mod tests { use legion::{IntoQuery, Resources, World}; diff --git a/crates/compiler/src/lowering/register_stages.rs b/crates/compiler/src/lowering/register_stages.rs index 1ec425e24a..1246a794a7 100644 --- a/crates/compiler/src/lowering/register_stages.rs +++ b/crates/compiler/src/lowering/register_stages.rs @@ -294,7 +294,7 @@ fn unknown_resource_diagnostic(resource_name: &ResourceName) -> Diagnostic<()> { .with_labels(vec![Label::primary((), resource_name.span())]) } -#[cfg(test)] +#[cfg(never)] mod tests { use indexmap::IndexMap; use legion::{IntoQuery, Resources, World}; diff --git a/crates/compiler/src/lowering/register_tensors.rs b/crates/compiler/src/lowering/register_tensors.rs index 167f03a382..2d560f1452 100644 --- a/crates/compiler/src/lowering/register_tensors.rs +++ b/crates/compiler/src/lowering/register_tensors.rs @@ -215,7 +215,7 @@ fn unknown_element_type_diagnostic(name: &str) -> Diagnostic<()> { .with_message(format!("Unknown element type, \"{}\"", name)) } -#[cfg(test)] +#[cfg(never)] mod tests { use std::str::FromStr; diff --git a/crates/compiler/src/parse/mod.rs b/crates/compiler/src/parse/mod.rs index 59ee62463c..12d6ec247d 100644 --- a/crates/compiler/src/parse/mod.rs +++ b/crates/compiler/src/parse/mod.rs @@ -1,72 +1,32 @@ -//! The parsing phase. -//! -//! This is a simple phase which just calls [`Document::parse()`] and stores -//! the resulting [`DocumentV1`] in the global [`legion::Resources`]. - mod yaml; -use codespan::Span; +use std::sync::Arc; + use codespan_reporting::diagnostic::{Diagnostic, Label}; -use legion::{systems::CommandBuffer, Registry}; pub use self::yaml::*; -use crate::{ - inputs::Inputs, phases::Phase, serialize::RegistryExt, BuildContext, - Diagnostics, -}; -pub fn phase() -> Phase { - Phase::with_setup(|res| { - res.insert(Diagnostics::new()); - }) - .and_then(run_system) +#[tracing::instrument(skip(src), err)] +pub fn parse(src: &str) -> Result { + Document::parse(src) + .map_err(|e| ParseFailedDiagnostic { inner: Arc::new(e) }) } -#[salsa::query_group(ParseGroup)] -pub trait Parse: Inputs { - #[salsa::dependencies] - fn parse(&self) -> Result>; +#[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)] +#[error("Unable to parse the Runefile: {}", inner)] +#[diagnostic(code("P001"))] +pub struct ParseFailedDiagnostic { + #[source] + inner: Arc, } -#[legion::system] -fn run( - cmd: &mut CommandBuffer, - #[resource] build_context: &BuildContext, - #[resource] diags: &mut Diagnostics, -) { - let src = &build_context.runefile; - - match Document::parse(src) { - Ok(d) => { - cmd.exec_mut(move |_, res| { - res.insert(d.clone().to_v1()); - }); - }, - Err(e) => { - diags.push(parse_failed_diagnostic(e)); - }, +impl ParseFailedDiagnostic { + pub fn as_codespan_diagnostic(&self) -> Diagnostic<()> { + let mut diag = Diagnostic::error().with_message(self.to_string()); + if let Some(location) = self.inner.location() { + let ix = location.index(); + diag = diag.with_labels(vec![Label::primary((), ix..ix)]); + } + diag } } - -fn parse(db: &dyn Parse) -> Result> { - let ctx = db.build_context(); - Document::parse(&ctx.runefile).map_err(parse_failed_diagnostic) -} - -fn parse_failed_diagnostic(e: serde_yaml::Error) -> Diagnostic<()> { - let msg = format!("Unable to parse the input: {}", e); - - let mut diag = Diagnostic::error().with_message(msg); - if let Some(location) = e.location() { - let ix = location.index(); - diag = diag.with_labels(vec![Label::primary((), ix..ix)]); - } - diag -} - -pub(crate) fn register_components(registry: &mut Registry) { - registry - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::(); -} diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index 622ebe8847..7d09b03bb6 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -447,7 +447,7 @@ impl Stage { /// Something that could be either a reference to a resource (`$resource`) /// or a plain string (`./path`). -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum ResourceOrString { Resource(ResourceName), String(String), @@ -731,6 +731,8 @@ impl ResourceDeclaration { Copy, Clone, PartialEq, + Eq, + Hash, serde::Serialize, serde::Deserialize, schemars::JsonSchema, diff --git a/crates/compiler/src/phases.rs b/crates/compiler/src/phases.rs index 58c4446a36..07cf0eaed4 100644 --- a/crates/compiler/src/phases.rs +++ b/crates/compiler/src/phases.rs @@ -9,7 +9,7 @@ use crate::{ hooks::{Continuation, Ctx, Hooks}, inputs::Inputs, lowering::{self, Name}, - parse::Parse, + parse::parse, type_check, BuildContext, Diagnostics, FeatureFlags, }; @@ -46,12 +46,13 @@ pub fn build_with_hooks( } log::debug!("Beginning the \"parse\" phase"); - match db.parse() { + match parse(db.build_context().runefile.as_str()) { Ok(d) => { res.insert(d.clone().to_v1()); }, Err(e) => { - res.get_mut_or_default::().push(e); + res.get_mut_or_default::() + .push(e.as_codespan_diagnostic()); }, } @@ -138,7 +139,6 @@ fn update_db_before_codegen(world: &World, db: &mut Database) { #[derive(Default)] #[salsa::database( - crate::parse::ParseGroup, crate::codegen::inputs::CodegenInputsGroup, crate::codegen::CodegenGroup, crate::compile::CompileGroup, diff --git a/crates/compiler/src/serialize.rs b/crates/compiler/src/serialize.rs index a1d47ef67a..7410b4d8d7 100644 --- a/crates/compiler/src/serialize.rs +++ b/crates/compiler/src/serialize.rs @@ -26,7 +26,6 @@ impl RegistryExt for Registry { pub fn registry() -> Registry { let mut registry = Registry::new(); - crate::parse::register_components(&mut registry); crate::lowering::register_components(&mut registry); crate::type_check::register_components(&mut registry); crate::codegen::register_components(&mut registry); From 697ec560cb097bc4c956f49d6830d31c512e7a35 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 19 Apr 2022 00:16:02 +0800 Subject: [PATCH 06/84] Copied across the parsing code --- Cargo.lock | 558 ++++++++++- crates/compiler-2/Cargo.toml | 27 + crates/compiler-2/README.md | 10 + crates/compiler-2/runefile-schema.json | 282 ++++++ crates/compiler-2/src/lib.rs | 9 + crates/compiler-2/src/macros.rs | 31 + crates/compiler-2/src/parse/mod.rs | 18 + crates/compiler-2/src/parse/yaml.rs | 1221 ++++++++++++++++++++++++ 8 files changed, 2144 insertions(+), 12 deletions(-) create mode 100644 crates/compiler-2/Cargo.toml create mode 100644 crates/compiler-2/README.md create mode 100644 crates/compiler-2/runefile-schema.json create mode 100644 crates/compiler-2/src/lib.rs create mode 100644 crates/compiler-2/src/macros.rs create mode 100644 crates/compiler-2/src/parse/mod.rs create mode 100644 crates/compiler-2/src/parse/yaml.rs diff --git a/Cargo.lock b/Cargo.lock index d913d583c0..7e102a5142 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -511,6 +511,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cranelift-bforest" version = "0.76.0" @@ -833,6 +849,15 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + [[package]] name = "enum-iterator" version = "0.7.0" @@ -912,6 +937,16 @@ dependencies = [ "regex", ] +[[package]] +name = "fancy-regex" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95b4efe5be9104a4a18a9916e86654319895138be727b229820c39257c30dda" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -948,6 +983,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -974,12 +1024,70 @@ dependencies = [ "num", ] +[[package]] +name = "fraction" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb65943183b6b3cbf00f64c181e8178217e30194381b150e4f87ec59864c803" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getopts" version = "0.2.21" @@ -997,7 +1105,7 @@ checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] @@ -1059,6 +1167,25 @@ dependencies = [ "regex", ] +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -1161,12 +1288,12 @@ dependencies = [ "im", "indexmap", "indoc", - "jsonschema", + "jsonschema 0.13.3", "legion", "log", "miette", "once_cell", - "pretty_assertions 1.2.0", + "pretty_assertions 1.2.1", "proc-macro2", "quote", "regex", @@ -1181,6 +1308,26 @@ dependencies = [ "zip", ] +[[package]] +name = "hotg-rune-compiler-2" +version = "0.0.0" +dependencies = [ + "im", + "indexmap", + "jsonschema 0.15.2", + "miette", + "once_cell", + "pretty_assertions 1.2.1", + "regex", + "salsa", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "thiserror", + "tracing", +] + [[package]] name = "hotg-rune-core" version = "0.11.3" @@ -1209,7 +1356,7 @@ version = "0.11.3" dependencies = [ "difference", "hotg-rune-core", - "pretty_assertions 1.2.0", + "pretty_assertions 1.2.1", "proc-macro2", "quote", "serde", @@ -1224,7 +1371,7 @@ dependencies = [ "difference", "hotg-rune-core", "hotg-rune-proc-block-macros", - "pretty_assertions 1.2.0", + "pretty_assertions 1.2.1", "serde", ] @@ -1279,6 +1426,40 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a164bb2ceaeff4f42542bdb847c41517c78a60f5649671b2a07312b6e117549" +[[package]] +name = "http" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.1", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "human-panic" version = "1.0.3" @@ -1300,6 +1481,43 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.1", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1353,9 +1571,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown 0.11.2", @@ -1380,6 +1598,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "iso8601" version = "0.4.1" @@ -1447,8 +1671,8 @@ dependencies = [ "anyhow", "base64", "bytecount", - "fancy-regex", - "fraction", + "fancy-regex 0.7.1", + "fraction 0.9.0", "iso8601", "itoa 1.0.1", "lazy_static", @@ -1464,6 +1688,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "jsonschema" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af26b80b2c3d68bd5b68d36160573f9d497cfe1cc81645a6820deed349d54f02" +dependencies = [ + "ahash", + "anyhow", + "base64", + "bytecount", + "fancy-regex 0.8.0", + "fraction 0.10.0", + "iso8601", + "itoa 1.0.1", + "lazy_static", + "memchr", + "num-cmp", + "parking_lot 0.12.0", + "percent-encoding", + "regex", + "reqwest", + "serde", + "serde_json", + "structopt", + "time 0.3.7", + "url", + "uuid", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1683,6 +1936,12 @@ dependencies = [ "syn", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1708,12 +1967,53 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mio" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + [[package]] name = "more-asserts" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.1" @@ -1730,6 +2030,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.2.1" @@ -1887,6 +2196,39 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_type" version = "2.4.0" @@ -1986,6 +2328,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.24" @@ -2082,9 +2430,9 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c038cb5319b9c704bf9c227c261d275bfec0ad438118a2787ce47944fb228b" +checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" dependencies = [ "ansi_term", "ctor", @@ -2364,6 +2712,42 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rkyv" version = "0.7.36" @@ -2479,6 +2863,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "schemars" version = "0.8.8" @@ -2528,6 +2922,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.11.0" @@ -2627,6 +3044,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa 1.0.1", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.8.23" @@ -2670,12 +3099,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + [[package]] name = "smallvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2851,7 +3296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", - "wasi", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] @@ -2897,6 +3342,46 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "tokio" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "winapi", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.8" @@ -2906,6 +3391,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + [[package]] name = "tracing" version = "0.1.34" @@ -2969,6 +3460,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -3095,12 +3592,28 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.79" @@ -3126,6 +3639,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.79" @@ -3506,6 +4031,15 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "xtask" version = "0.0.0" diff --git a/crates/compiler-2/Cargo.toml b/crates/compiler-2/Cargo.toml new file mode 100644 index 0000000000..2261635f79 --- /dev/null +++ b/crates/compiler-2/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "hotg-rune-compiler-2" +version = "0.0.0" +edition = "2021" +publish = false +license = "MIT OR Apache-2.0" +readme = "README.md" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +im = { version = "15.0.0", features = ["serde"] } +indexmap = { version = "1.8.1", features = ["serde-1"] } +miette = "4.5.0" +once_cell = "1.10.0" +regex = "1.5.5" +salsa = "0.16.1" +schemars = { version = "0.8.8", features = ["indexmap"] } +serde = "1.0.136" +serde_json = "1.0.79" +serde_yaml = "0.8.23" +thiserror = "1.0.30" +tracing = "0.1.34" + +[dev-dependencies] +jsonschema = "0.15.2" +pretty_assertions = "1.2.1" diff --git a/crates/compiler-2/README.md b/crates/compiler-2/README.md new file mode 100644 index 0000000000..2b85b98df5 --- /dev/null +++ b/crates/compiler-2/README.md @@ -0,0 +1,10 @@ +# The Rune Compiler + +A compiler that compiles your data processing pipeline into a portable +WebAssembly binary. + +## Architecture + +The compiler is split into 4 + +- [`crate::parse`] - parser diff --git a/crates/compiler-2/runefile-schema.json b/crates/compiler-2/runefile-schema.json new file mode 100644 index 0000000000..89b176dbcf --- /dev/null +++ b/crates/compiler-2/runefile-schema.json @@ -0,0 +1,282 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "anyOf": [ + { + "$ref": "#/definitions/DocumentV1" + } + ], + "definitions": { + "Argument": { + "anyOf": [ + { + "$ref": "#/definitions/ResourceName" + }, + { + "type": "string" + }, + { + "type": "number" + } + ], + "description": "Something that could be either a reference to a resource (`$resource`) or a plain string (`./path`)." + }, + "CapabilityStage": { + "description": "A stage which reads inputs from the runtime.", + "properties": { + "args": { + "additionalProperties": { + "$ref": "#/definitions/Argument" + }, + "type": "object" + }, + "capability": { + "description": "What type of capability to use (\"IMAGE\", \"SOUND\", etc.).", + "type": "string" + }, + "outputs": { + "items": { + "$ref": "#/definitions/Type" + }, + "type": "array" + } + }, + "required": [ + "capability" + ], + "type": "object" + }, + "DocumentV1": { + "description": "Version 1 of the `Runefile.yml` format.", + "properties": { + "image": { + "allOf": [ + { + "$ref": "#/definitions/Path" + } + ], + "description": "The base image that defines the interface between a Rune and its runtime.\n\nThis should always be `\"runicos/base\"`." + }, + "pipeline": { + "additionalProperties": { + "$ref": "#/definitions/Stage" + }, + "description": "The various stages in the Runefile's pipeline.", + "type": "object" + }, + "resources": { + "additionalProperties": { + "$ref": "#/definitions/ResourceDeclaration" + }, + "default": {}, + "description": "Any resources that can be accessed by pipeline stages.", + "type": "object" + }, + "version": { + "description": "The version number. Must always be `\"1\"`.", + "format": "uint", + "maximum": 1.0, + "minimum": 1.0, + "type": "integer" + } + }, + "required": [ + "image", + "pipeline", + "version" + ], + "type": "object" + }, + "Input": { + "description": "\nThe name of a tensor.\n\nTypically something like \"stage\", or \"stage.2\" if the stage has multiple outputs.\n", + "format": "string", + "pattern": "^(?P[a-zA-Z_][\\w-]*)(?:\\.(?P\\d+))?$", + "type": "string" + }, + "ModelStage": { + "description": "A ML model which will be executed by the runtime.", + "properties": { + "args": { + "additionalProperties": { + "$ref": "#/definitions/Argument" + }, + "type": "object" + }, + "inputs": { + "description": "Tensors to use as input to this model.", + "items": { + "$ref": "#/definitions/Input" + }, + "type": "array" + }, + "model": { + "anyOf": [ + { + "$ref": "#/definitions/ResourceName" + }, + { + "type": "string" + } + ], + "description": "The model to use, or a resource which specifies the model to use." + }, + "outputs": { + "description": "The tensors that this model outputs.", + "items": { + "$ref": "#/definitions/Type" + }, + "type": "array" + } + }, + "required": [ + "model" + ], + "type": "object" + }, + "OutStage": { + "description": "A stage which passes outputs back to the runtime.", + "properties": { + "args": { + "additionalProperties": { + "$ref": "#/definitions/Argument" + }, + "type": "object" + }, + "inputs": { + "items": { + "$ref": "#/definitions/Input" + }, + "type": "array" + }, + "out": { + "description": "The type of output (e.g. \"SERIAL\").", + "type": "string" + } + }, + "required": [ + "out" + ], + "type": "object" + }, + "Path": { + "description": "\nA specification for finding a dependency.\n\nThe full syntax is `base@version#sub_path` where\n\n- `base` is a URL or the name of a repository on GitHub (e.g. `hotg-ai/rune`\n or `https://github.com/hotg-ai/rune`)\n- `version` is an optional field specifying the version (e.g. as a git tag)\n- `sub_path` is an optional field which is useful when pointing to\n repositories with multiple relevant items because it lets you specify\n which directory the specified item is in.\n", + "format": "string", + "pattern": "(?x)\n (?P[\\w\\d:/_.-]+)\n (?:@(?P[\\w\\d./-]+))?\n (?:\\#(?P[\\w\\d._/-]+))?\n ", + "type": "string" + }, + "ProcBlockStage": { + "description": "A stage which executes a procedural block.", + "properties": { + "args": { + "additionalProperties": { + "$ref": "#/definitions/Argument" + }, + "type": "object" + }, + "inputs": { + "items": { + "$ref": "#/definitions/Input" + }, + "type": "array" + }, + "outputs": { + "items": { + "$ref": "#/definitions/Type" + }, + "type": "array" + }, + "proc-block": { + "description": "A [`Path`] that Rune can use to locate the proc block.", + "format": "string", + "pattern": "(?x)\n (?P[\\w\\d:/_.-]+)\n (?:@(?P[\\w\\d./-]+))?\n (?:\\#(?P[\\w\\d._/-]+))?\n ", + "type": "string" + } + }, + "required": [ + "proc-block" + ], + "type": "object" + }, + "ResourceDeclaration": { + "additionalProperties": false, + "description": "The declaration for a resource, typically something like a wordlist or environment variable.", + "properties": { + "inline": { + "description": "A resource who's default value is specified inline.", + "type": [ + "string", + "null" + ] + }, + "path": { + "description": "A resource who's default value is meant to be loaded from a file.", + "type": [ + "string", + "null" + ] + }, + "type": { + "allOf": [ + { + "$ref": "#/definitions/ResourceType" + } + ], + "default": "string" + } + }, + "type": "object" + }, + "ResourceName": { + "description": "\nA reference to some [`ResourceDeclaration`]. It typically looks like\n`$RESOURCE_NAME`.\n", + "format": "string", + "pattern": "^\\$[_a-zA-Z][_a-zA-Z0-9]*$", + "type": "string" + }, + "ResourceType": { + "description": "How the resource should be treated inside the Rune.", + "enum": [ + "string", + "binary" + ], + "type": "string" + }, + "Stage": { + "anyOf": [ + { + "$ref": "#/definitions/ModelStage" + }, + { + "$ref": "#/definitions/ProcBlockStage" + }, + { + "$ref": "#/definitions/CapabilityStage" + }, + { + "$ref": "#/definitions/OutStage" + } + ], + "description": "A stage in the Rune's pipeline." + }, + "Type": { + "description": "The element type and dimensions for a particular tensor.", + "properties": { + "dimensions": { + "items": { + "format": "uint", + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + }, + "type": { + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + } + }, + "description": "The top level Runefile type.", + "title": "Document" +} \ No newline at end of file diff --git a/crates/compiler-2/src/lib.rs b/crates/compiler-2/src/lib.rs new file mode 100644 index 0000000000..0534184ec6 --- /dev/null +++ b/crates/compiler-2/src/lib.rs @@ -0,0 +1,9 @@ +#![doc= include_str!("../README.md")] + +#[macro_use] +mod macros; +pub mod parse; + +#[cfg(test)] +#[macro_use] +extern crate pretty_assertions; diff --git a/crates/compiler-2/src/macros.rs b/crates/compiler-2/src/macros.rs new file mode 100644 index 0000000000..ebdc3f08d2 --- /dev/null +++ b/crates/compiler-2/src/macros.rs @@ -0,0 +1,31 @@ +#[cfg(test)] +macro_rules! map { + // map-like + ($($k:ident : $v:expr),* $(,)?) => { + std::iter::Iterator::collect(IntoIterator::into_iter([ + $( + (String::from(stringify!($k)), $v) + ),* + ])) + }; + // set-like + ($($v:expr),* $(,)?) => { + std::iter::Iterator::collect(std::array::IntoIter::new([$($v,)*])) + }; +} + +#[cfg(test)] +macro_rules! ty { + ($type:ident [$($dim:expr),*]) => { + crate::parse::Type { + name: String::from(stringify!($type)), + dimensions: vec![ $($dim),*], + } + }; + ($type:ident) => { + crate::parse::Type { + name: String::from(stringify!($type)), + dimensions: vec![], + } + } + } diff --git a/crates/compiler-2/src/parse/mod.rs b/crates/compiler-2/src/parse/mod.rs new file mode 100644 index 0000000000..5f5f34d263 --- /dev/null +++ b/crates/compiler-2/src/parse/mod.rs @@ -0,0 +1,18 @@ +mod yaml; + +use std::sync::Arc; + +pub use self::yaml::*; + +#[tracing::instrument(skip(src), err)] +pub fn parse(src: &str) -> Result { + Document::parse(src) + .map_err(|e| ParseFailedDiagnostic { inner: Arc::new(e) }) +} + +#[derive(Debug, Clone, thiserror::Error)] +#[error("Unable to parse the Runefile: {}", inner)] +pub struct ParseFailedDiagnostic { + #[source] + inner: Arc, +} diff --git a/crates/compiler-2/src/parse/yaml.rs b/crates/compiler-2/src/parse/yaml.rs new file mode 100644 index 0000000000..187d5b8037 --- /dev/null +++ b/crates/compiler-2/src/parse/yaml.rs @@ -0,0 +1,1221 @@ +//! Definitions for the Runefile's YAML format. + +use std::{ + borrow::Cow, + fmt::{self, Display, Formatter}, + ops::Deref, + str::FromStr, +}; + +use indexmap::IndexMap; +use once_cell::sync::Lazy; +use regex::Regex; +use schemars::{ + gen::SchemaGenerator, + schema::{ + InstanceType, Metadata, Schema, SchemaObject, SubschemaValidation, + }, + JsonSchema, +}; +use serde::{ + de::{Deserialize, Deserializer, Error as _}, + ser::{Serialize, Serializer}, +}; + +static RESOURCE_NAME_PATTERN: Lazy = + Lazy::new(|| Regex::new(r"^\$[_a-zA-Z][_a-zA-Z0-9]*$").unwrap()); + +/// The top level Runefile type. +#[derive(Debug, Clone, PartialEq, JsonSchema)] +#[schemars(untagged)] +pub enum Document { + V1(DocumentV1), +} + +impl Document { + pub fn to_v1(self) -> DocumentV1 { + match self { + Document::V1(d) => d, + } + } +} + +impl From for Document { + fn from(v1: DocumentV1) -> Self { Document::V1(v1) } +} + +mod document_serde { + use serde::de::Unexpected; + use serde_yaml::Value; + + use super::*; + + #[derive(serde::Serialize, serde::Deserialize)] + struct Repr { + version: usize, + #[serde(flatten)] + inner: T, + } + + impl Repr { + fn new(version: usize, inner: T) -> Self { Repr { version, inner } } + } + + impl Serialize for Document { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Document::V1(v1) => Repr::new(1, v1).serialize(serializer), + } + } + } + + impl<'de> Deserialize<'de> for Document { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + let version_key = Value::from("version"); + let version = value + .as_mapping() + .and_then(|m| m.get(&version_key)) + .and_then(|v| v.as_u64()); + + match version { + Some(1) => { + let v1: DocumentV1 = serde_yaml::from_value(value) + .map_err(D::Error::custom)?; + Ok(Document::V1(v1)) + }, + Some(other) => Err(D::Error::invalid_value( + Unexpected::Unsigned(other), + &"version to be 1", + )), + None => Err(D::Error::missing_field("version")), + } + } + } +} + +macro_rules! impl_json_schema_via_regex { + ($ty:ty, $pattern:expr, $docs:literal) => { + impl JsonSchema for $ty { + fn schema_name() -> String { String::from(stringify!($ty)) } + + fn json_schema(_: &mut SchemaGenerator) -> Schema { + let mut schema = SchemaObject { + instance_type: Some(InstanceType::String.into()), + format: Some(String::from("string")), + metadata: Some(Box::new(Metadata { + description: Some(String::from($docs)), + ..Default::default() + })), + ..Default::default() + }; + + schema.string().pattern = Some($pattern.to_string()); + + schema.into() + } + } + }; +} + +/// Version 1 of the `Runefile.yml` format. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct DocumentV1 { + /// The version number. Must always be `"1"`. + #[schemars(required, range(min = 1, max = 1))] + pub version: usize, + /// The base image that defines the interface between a Rune and its + /// runtime. + /// + /// This should always be `"runicos/base"`. + pub image: Image, + /// The various stages in the Runefile's pipeline. + pub pipeline: IndexMap, + /// Any resources that can be accessed by pipeline stages. + #[serde(default)] + pub resources: IndexMap, +} + +impl Document { + pub fn parse(yaml: &str) -> Result { + serde_yaml::from_str(yaml) + } + + pub fn write_as_yaml(&self, writer: W) -> Result<(), serde_yaml::Error> + where + W: std::io::Write, + { + serde_yaml::to_writer(writer, self)?; + Ok(()) + } +} + +impl FromStr for Document { + type Err = serde_yaml::Error; + + fn from_str(s: &str) -> Result { Document::parse(s) } +} + +/// A specification for finding a dependency. +/// +/// The full syntax is `base@version#sub_path` where +/// +/// - `base` is a URL or the name of a repository on GitHub (e.g. `hotg-ai/rune` +/// or `https://github.com/hotg-ai/rune`) +/// - `version` is an optional field specifying the version (e.g. as a git tag) +/// - `sub_path` is an optional field which is useful when pointing to +/// repositories with multiple relevant items because it lets you specify +/// which directory the specified item is in. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Path { + pub base: String, + pub sub_path: Option, + pub version: Option, +} + +impl_json_schema_via_regex!( + Path, + PATH_PATTERN, + r#" +A specification for finding a dependency. + +The full syntax is `base@version#sub_path` where + +- `base` is a URL or the name of a repository on GitHub (e.g. `hotg-ai/rune` + or `https://github.com/hotg-ai/rune`) +- `version` is an optional field specifying the version (e.g. as a git tag) +- `sub_path` is an optional field which is useful when pointing to + repositories with multiple relevant items because it lets you specify + which directory the specified item is in. +"# +); + +impl Path { + pub fn new( + base: impl Into, + sub_path: impl Into>, + version: impl Into>, + ) -> Self { + Path { + base: base.into(), + sub_path: sub_path.into(), + version: version.into(), + } + } +} + +impl Display for Path { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let Path { + base, + sub_path, + version, + } = self; + + write!(f, "{}", base)?; + if let Some(version) = version { + write!(f, "@{}", version)?; + } + if let Some(sub) = sub_path { + write!(f, "#{}", sub)?; + } + + Ok(()) + } +} + +static PATH_PATTERN: Lazy = Lazy::new(|| { + Regex::new( + r"(?x) + (?P[\w\d:/_.-]+) + (?:@(?P[\w\d./-]+))? + (?:\#(?P[\w\d._/-]+))? + ", + ) + .unwrap() +}); + +impl FromStr for Path { + type Err = PathParseError; + + fn from_str(s: &str) -> Result { + let captures = PATH_PATTERN.captures(s).ok_or(PathParseError)?; + + let base = captures["base"].to_string(); + let version = captures.name("version").map(|m| m.as_str().to_string()); + let sub_path = + captures.name("sub_path").map(|m| m.as_str().to_string()); + + Ok(Path { + base, + version, + sub_path, + }) + } +} + +impl Serialize for Path { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Path { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = Cow::<'de, str>::deserialize(deserializer)?; + + s.parse().map_err(D::Error::custom) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Default)] +pub struct PathParseError; + +impl Display for PathParseError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "Unable to parse the path") + } +} + +impl std::error::Error for PathParseError {} + +/// A ML model which will be executed by the runtime. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct ModelStage { + /// The model to use, or a resource which specifies the model to use. + #[schemars(required)] + pub model: ResourceOrString, + /// Tensors to use as input to this model. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inputs: Vec, + /// The tensors that this model outputs. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub outputs: Vec, + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub args: IndexMap, +} + +/// A stage which executes a procedural block. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct ProcBlockStage { + /// A [`Path`] that Rune can use to locate the proc block. + #[serde(rename = "proc-block")] + #[schemars(required)] + pub proc_block: Path, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inputs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub outputs: Vec, + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub args: IndexMap, +} + +/// A stage which reads inputs from the runtime. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct CapabilityStage { + /// What type of capability to use ("IMAGE", "SOUND", etc.). + #[schemars(required)] + pub capability: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub outputs: Vec, + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub args: IndexMap, +} + +/// A stage which passes outputs back to the runtime. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct OutStage { + /// The type of output (e.g. "SERIAL"). + #[schemars(required)] + pub out: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inputs: Vec, + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub args: IndexMap, +} + +/// A stage in the Rune's pipeline. +#[derive( + Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize, JsonSchema, +)] +#[serde(untagged, rename_all = "kebab-case")] +pub enum Stage { + Model(ModelStage), + ProcBlock(ProcBlockStage), + Capability(CapabilityStage), + Out(OutStage), +} + +impl Stage { + pub fn inputs(&self) -> &[Input] { + match self { + Stage::Model(ModelStage { inputs, .. }) + | Stage::ProcBlock(ProcBlockStage { inputs, .. }) + | Stage::Out(OutStage { inputs, .. }) => inputs, + Stage::Capability(_) => &[], + } + } + + pub fn inputs_mut(&mut self) -> Option<&mut Vec> { + match self { + Stage::Model(ModelStage { inputs, .. }) + | Stage::ProcBlock(ProcBlockStage { inputs, .. }) + | Stage::Out(OutStage { inputs, .. }) => Some(inputs), + Stage::Capability(_) => None, + } + } + + pub fn output_type(&self) -> Option<&Type> { + match self.output_types() { + [] => None, + [output] => Some(output), + _ => unimplemented!("Multiple outputs aren't supported yet"), + } + } + + pub fn output_types(&self) -> &[Type] { + match self { + Stage::Model(ModelStage { outputs, .. }) + | Stage::ProcBlock(ProcBlockStage { outputs, .. }) + | Stage::Capability(CapabilityStage { outputs, .. }) => outputs, + Stage::Out(OutStage { .. }) => &[], + } + } + + pub fn args(&self) -> &IndexMap { + match self { + Stage::Model(m) => &m.args, + Stage::ProcBlock(p) => &p.args, + Stage::Capability(c) => &c.args, + Stage::Out(out) => &out.args, + } + } +} + +/// Something that could be either a reference to a resource (`$resource`) +/// or a plain string (`./path`). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ResourceOrString { + Resource(ResourceName), + String(String), +} + +impl JsonSchema for ResourceOrString { + fn schema_name() -> std::string::String { "ResourceOrString".to_owned() } + + fn json_schema(gen: &mut SchemaGenerator) -> Schema { + let resource_name = gen.subschema_for::(); + let string = gen.subschema_for::(); + + let description = "Something that could be either a reference to a \ + resource (`$resource`) or a plain string \ + (`./path`)."; + + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + description: Some(description.to_owned()), + ..Default::default() + })), + subschemas: Some(Box::new(SubschemaValidation { + any_of: Some(vec![resource_name, string]), + ..Default::default() + })), + ..Default::default() + }) + } +} + +impl Serialize for ResourceOrString { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ResourceOrString { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = ResourceOrString; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a number, string, or \"$RESOURCE_NAME\"") + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(ResourceOrString::String(v.to_string())) + } + + fn visit_i64(self, v: i64) -> Result + where + E: serde::de::Error, + { + Ok(ResourceOrString::String(v.to_string())) + } + + fn visit_f64(self, v: f64) -> Result + where + E: serde::de::Error, + { + Ok(ResourceOrString::String(v.to_string())) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let v = v.trim(); + + if !v.starts_with('$') { + return Ok(ResourceOrString::String(v.to_string())); + } + + match ResourceName::from_str(v) { + Ok(name) => Ok(ResourceOrString::Resource(name)), + Err(e) => Err(E::custom(e)), + } + } + + fn visit_seq(self, _: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + Err(A::Error::custom("lists aren't supported")) + } + } + + deserializer.deserialize_any(Visitor) + } +} + +impl Display for ResourceOrString { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + ResourceOrString::String(path) => write!(f, "{}", path), + ResourceOrString::Resource(res) => write!(f, "{}", res), + } + } +} + +impl> From for ResourceOrString { + fn from(s: S) -> Self { ResourceOrString::String(s.into()) } +} + +impl From for ResourceOrString { + fn from(name: ResourceName) -> Self { ResourceOrString::Resource(name) } +} + +/// A newtype around [`ResourceOrString`] which is used in each stage's `args` +/// dictionary. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +pub struct Argument(pub ResourceOrString); + +impl JsonSchema for Argument { + fn schema_name() -> std::string::String { "Argument".to_owned() } + + fn json_schema(gen: &mut SchemaGenerator) -> Schema { + let number = gen.subschema_for::(); + + let mut schema = ResourceOrString::json_schema(gen).into_object(); + schema.subschemas().any_of.as_mut().unwrap().push(number); + + schema.into() + } +} + +impl> From for Argument { + fn from(value: T) -> Self { Argument(value.into()) } +} + +impl Deref for Argument { + type Target = ResourceOrString; + + fn deref(&self) -> &Self::Target { &self.0 } +} + +/// The element type and dimensions for a particular tensor. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +pub struct Type { + #[serde(rename = "type")] + pub name: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dimensions: Vec, +} + +/// The name of a tensor. +/// +/// Typically something like "stage", or "stage.2" if the stage has multiple +/// outputs. +#[derive(Debug, Clone, PartialEq, Hash, Eq, Ord, PartialOrd)] +pub struct Input { + pub name: String, + pub index: Option, +} + +impl_json_schema_via_regex!( + Input, + INPUT_PATTERN, + r#" +The name of a tensor. + +Typically something like "stage", or "stage.2" if the stage has multiple outputs. +"# +); + +impl Input { + pub fn new( + name: impl Into, + index: impl Into>, + ) -> Self { + Input { + name: name.into(), + index: index.into(), + } + } +} + +static INPUT_PATTERN: Lazy = Lazy::new(|| { + Regex::new(r"^(?P[a-zA-Z_][\w-]*)(?:\.(?P\d+))?$").unwrap() +}); + +impl FromStr for Input { + type Err = Box; + + fn from_str(s: &str) -> Result { + let captures = INPUT_PATTERN + .captures(s) + .ok_or("Expected something like \"fft\" or \"fft.2\"")?; + + let name = &captures["name"]; + let index = captures.name("index").map(|m| { + m.as_str() + .parse::() + .expect("Guaranteed by the regex") + }); + + Ok(Input::new(name, index)) + } +} + +impl Display for Input { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self.index { + Some(index) => write!(f, "{}.{}", self.name, index), + None => write!(f, "{}", self.name), + } + } +} + +impl Serialize for Input { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Input { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let raw = Cow::::deserialize(deserializer)?; + Input::from_str(&raw).map_err(|e| D::Error::custom(e.to_string())) + } +} + +/// The declaration for a resource, typically something like a wordlist or +/// environment variable. +#[derive( + Debug, + Clone, + Default, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +#[serde(deny_unknown_fields)] +pub struct ResourceDeclaration { + /// A resource who's default value is specified inline. + pub inline: Option, + /// A resource who's default value is meant to be loaded from a file. + pub path: Option, + #[serde(rename = "type", default)] + pub ty: ResourceType, +} + +/// How the resource should be treated inside the Rune. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +#[serde(rename_all = "kebab-case")] +pub enum ResourceType { + /// The resource should be treated like as a `&str`. + String, + /// The resource should be treated like a `&[u8]`. + Binary, +} + +impl Default for ResourceType { + fn default() -> Self { ResourceType::String } +} + +/// A reference to some [`ResourceDeclaration`]. It typically looks like +/// `$RESOURCE_NAME`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ResourceName(pub String); + +impl_json_schema_via_regex!( + ResourceName, + RESOURCE_NAME_PATTERN, + r#" +A reference to some [`ResourceDeclaration`]. It typically looks like +`$RESOURCE_NAME`. +"# +); + +impl> From for ResourceName { + fn from(s: S) -> Self { ResourceName(s.into()) } +} + +impl FromStr for ResourceName { + type Err = Box; + + fn from_str(s: &str) -> Result { + if !s.starts_with('$') { + return Err("resource names always start with a \"$\"".into()); + } + + if !RESOURCE_NAME_PATTERN.is_match(s) { + return Err("should be a valid identifier".into()); + } + + Ok(ResourceName(s[1..].to_string())) + } +} + +impl Deref for ResourceName { + type Target = String; + + fn deref(&self) -> &Self::Target { &self.0 } +} + +impl Serialize for ResourceName { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ResourceName { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let repr = Cow::::deserialize(deserializer)?; + + if !repr.starts_with('$') { + return Err(D::Error::custom( + "resource names always start with a \"$\"", + )); + } + + let name = &repr[1..]; + + if name.is_empty() { + Err(D::Error::custom("the resource name is empty")) + } else if !RESOURCE_NAME_PATTERN.is_match(name) { + Err(D::Error::custom("should be a valid identifier")) + } else { + Ok(ResourceName(name.to_string())) + } + } +} + +impl Display for ResourceName { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "${}", self.0) + } +} + +/// The image a Rune is based on. +#[derive( + Debug, + Clone, + PartialEq, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +#[schemars(transparent)] +pub struct Image(pub Path); + +impl FromStr for Image { + type Err = PathParseError; + + fn from_str(s: &str) -> Result { + Path::from_str(s).map(Image) + } +} + +#[cfg(test)] +mod tests { + use jsonschema::JSONSchema; + + use super::*; + + #[test] + fn parse_normal_input_specifier() { + let src = "audio"; + let should_be = Input::new("audio", None); + + let got = Input::from_str(src).unwrap(); + + assert_eq!(got, should_be); + assert_eq!(got.to_string(), src); + } + + #[test] + fn input_specifier_with_tuple() { + let src = "audio.2"; + let should_be = Input::new("audio", 2); + + let got = Input::from_str(src).unwrap(); + + assert_eq!(got, should_be); + assert_eq!(got.to_string(), src); + } + + #[test] + fn parse_paths() { + let inputs = vec![ + ("asdf", Path::new("asdf", None, None)), + ("runicos/base", Path::new("runicos/base", None, None)), + ( + "runicos/base@0.1.2", + Path::new("runicos/base", None, Some(String::from("0.1.2"))), + ), + ( + "runicos/base@latest", + Path::new("runicos/base", None, Some(String::from("latest"))), + ), + ( + "https://github.com/hotg-ai/rune", + Path::new("https://github.com/hotg-ai/rune", None, None), + ), + ( + "https://github.com/hotg-ai/rune@2", + Path::new( + "https://github.com/hotg-ai/rune", + None, + Some(String::from("2")), + ), + ), + ( + "hotg-ai/rune@v1.2#proc_blocks/normalize", + Path::new( + "hotg-ai/rune", + "proc_blocks/normalize".to_string(), + "v1.2".to_string(), + ), + ), + // Note: GitHub provides these refs that you can use as well as the + // normal tags and commits + ( + "hotg-ai/proc-blocks@refs/heads/master#normalize", + Path::new( + "hotg-ai/proc-blocks", + "normalize".to_string(), + "refs/heads/master".to_string(), + ), + ), + ]; + + for (src, should_be) in inputs { + let got: Path = src.parse().unwrap(); + assert_eq!(got, should_be, "{}", src); + let round_tripped = got.to_string(); + assert_eq!(round_tripped, src); + } + } + + #[test] + fn parse_v1() { + let src = "version: 1\nimage: asdf\npipeline: {}"; + + let got = Document::parse(src).unwrap(); + + assert!(matches!(got, Document::V1 { .. })); + } + + #[test] + #[should_panic = "expected version to be 1"] + fn other_versions_are_an_error() { + let src = "image: asdf\nversion: 2\npipeline:"; + + let got = Document::parse(src).unwrap(); + + assert!(matches!(got, Document::V1 { .. })); + } + + #[test] + fn inline_resource() { + let src = "inline: some data"; + let should_be = ResourceDeclaration { + inline: Some(String::from("some data")), + ..Default::default() + }; + + let got: ResourceDeclaration = serde_yaml::from_str(src).unwrap(); + + assert_eq!(got, should_be); + } + + #[test] + fn resource_from_disk() { + let src = "path: ./input.txt"; + let should_be = ResourceDeclaration { + path: Some(String::from("./input.txt")), + ..Default::default() + }; + + let got: ResourceDeclaration = serde_yaml::from_str(src).unwrap(); + + assert_eq!(got, should_be); + } + + #[test] + fn resource_with_no_default_value() { + let src = "resource_name: {}"; + let should_be = ResourceDeclaration::default(); + + let got: IndexMap = + serde_yaml::from_str(src).unwrap(); + + let declaration = &got[0]; + assert_eq!(declaration, &should_be); + } + + #[test] + fn model_name_from_resource() { + let src = "$MODEL"; + let should_be = ResourceOrString::Resource("MODEL".into()); + + let got: ResourceOrString = serde_yaml::from_str(src).unwrap(); + + assert_eq!(got, should_be); + + let round_tripped = serde_yaml::to_string(&got).unwrap(); + assert_eq!(round_tripped, "---\n$MODEL\n"); + } + + #[test] + #[should_panic = "should be a valid identifier"] + fn model_name_from_resource_must_not_be_empty() { + let src = "$"; + + let _: ResourceOrString = serde_yaml::from_str(src).unwrap(); + } + + #[test] + #[should_panic = "should be a valid identifier"] + fn model_name_from_resource_must_be_valid_identifier() { + let src = "$"; + + let _: ResourceOrString = serde_yaml::from_str(src).unwrap(); + } + + #[test] + fn model_name_from_path() { + let src = "./path"; + let should_be = ResourceOrString::String(String::from(src)); + + let got: ResourceOrString = serde_yaml::from_str(src).unwrap(); + + assert_eq!(got, should_be); + + let round_tripped = serde_yaml::to_string(&got).unwrap(); + assert_eq!(round_tripped, "---\n\"./path\"\n"); + } + + #[test] + fn proc_block_with_resource_for_arg() { + let src = r#" + some-proc-block: + proc-block: normalize + outputs: + - type: u8 + dimensions: [1] + args: + word-list: $WORD_LIST + "#; + let should_be = Stage::ProcBlock(ProcBlockStage { + proc_block: "normalize".parse().unwrap(), + inputs: Vec::new(), + outputs: vec![Type { + name: String::from("u8"), + dimensions: vec![1], + }], + args: vec![( + "word-list".to_string(), + ResourceName::from_str("$WORD_LIST").unwrap().into(), + )] + .into_iter() + .collect(), + }); + + let got: IndexMap = serde_yaml::from_str(src).unwrap(); + + let got = &got["some-proc-block"]; + assert_eq!(got, &should_be); + } + + #[test] + fn parse_yaml_pipeline() { + let src = r#" +version: 1 +image: "runicos/base" + +pipeline: + audio: + capability: SOUND + outputs: + - type: i16 + dimensions: [16000] + args: + hz: 16000 + + fft: + proc-block: "hotg-ai/rune#proc_blocks/fft" + inputs: + - audio + outputs: + - type: i8 + dimensions: [1960] + + model: + model: "./model.tflite" + inputs: + - fft + outputs: + - type: i8 + dimensions: [6] + + label: + proc-block: "hotg-ai/rune#proc_blocks/ohv_label" + inputs: + - model + outputs: + - type: utf8 + args: + labels: | + silence + unknown + up + down + left + right + + output: + out: SERIAL + inputs: + - label + "#; + let should_be = Document::V1(DocumentV1 { + version: 1, + image: "runicos/base".parse().unwrap(), + pipeline: map! { + audio: Stage::Capability(CapabilityStage { + capability: String::from("SOUND"), + outputs: vec![ty!(i16[16000])], + args: map! { hz: "16000".into() }, + }), + fft: Stage::ProcBlock(ProcBlockStage { + proc_block: "hotg-ai/rune#proc_blocks/fft".parse().unwrap(), + inputs: vec!["audio".parse().unwrap()], + outputs: vec![ty!(i8[1960])], + args: IndexMap::new(), + }), + model: Stage::Model(ModelStage { + model: "./model.tflite".into(), + inputs: vec!["fft".parse().unwrap()], + outputs: vec![ty!(i8[6])], + args: IndexMap::new(), + }), + label: Stage::ProcBlock(ProcBlockStage { + proc_block: "hotg-ai/rune#proc_blocks/ohv_label".parse().unwrap(), + inputs: vec!["model".parse().unwrap()], + outputs: vec![Type { name: String::from("utf8"), dimensions: Vec::new() }], + args: map! { + labels: "silence\nunknown\nup\ndown\nleft\nright".into() + }, + }), + output: Stage::Out(OutStage { + out: String::from("SERIAL"), + args: IndexMap::new(), + inputs: vec!["label".parse().unwrap()], + }), + }, + resources: map![], + }); + + let got = Document::parse(src).unwrap(); + + assert_eq!(got, should_be); + } + + #[test] + fn parse_audio_block() { + let src = r#" + capability: SOUND + outputs: + - type: i16 + dimensions: [16000] + args: + hz: 16000 + "#; + let should_be = Stage::Capability(CapabilityStage { + capability: String::from("SOUND"), + outputs: vec![Type { + name: String::from("i16"), + dimensions: vec![16000], + }], + args: map! { hz: "16000".into() }, + }); + + let got: Stage = serde_yaml::from_str(src).unwrap(); + + assert_eq!(got, should_be); + } + + #[test] + fn schema_is_in_sync_with_version_on_disk() { + let filename = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("runefile-schema.json"); + let existing_schema = std::fs::read_to_string(&filename).unwrap(); + let existing_schema: serde_json::Value = + serde_json::from_str(&existing_schema).unwrap(); + + let schema = schemars::schema_for!(Document); + let current_schema = serde_json::to_value(&schema).unwrap(); + + if existing_schema != current_schema { + let serialized = + serde_json::to_string_pretty(¤t_schema).unwrap(); + std::fs::write(&filename, serialized.as_bytes()).unwrap(); + panic!("The runefile-schema.json was out of date"); + } + } + + #[track_caller] + fn handle_errors<'a>( + errors: impl Iterator>, + ) -> ! { + for err in errors { + println!("{}", err); + } + + panic!("Validation failed"); + } + + #[test] + fn argument_schema_is_valid() { + let schema = schemars::schema_for!(Argument); + let schema_json = serde_json::to_value(&schema).unwrap(); + let compiled_schema = + JSONSchema::options().compile(&schema_json).unwrap(); + + let string = serde_json::Value::String("".to_string()); + compiled_schema + .validate(&string) + .unwrap_or_else(|e| handle_errors(e)); + + let resource = serde_json::Value::String("$resource".to_string()); + compiled_schema + .validate(&resource) + .unwrap_or_else(|e| handle_errors(e)); + + let number = serde_json::Value::Number(10.into()); + compiled_schema + .validate(&number) + .unwrap_or_else(|e| handle_errors(e)); + } +} From c6a5412716099901e4f4da0e01df743bb73dc5ef Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 19 Apr 2022 01:34:52 +0800 Subject: [PATCH 07/84] Wired up text and diagnostics --- crates/compiler-2/README.md | 11 ++- crates/compiler-2/src/diagnostics/mod.rs | 115 +++++++++++++++++++++++ crates/compiler-2/src/lib.rs | 11 ++- crates/compiler-2/src/parse/mod.rs | 18 +++- crates/compiler-2/src/text.rs | 44 +++++++++ 5 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 crates/compiler-2/src/diagnostics/mod.rs create mode 100644 crates/compiler-2/src/text.rs diff --git a/crates/compiler-2/README.md b/crates/compiler-2/README.md index 2b85b98df5..c7b1a08918 100644 --- a/crates/compiler-2/README.md +++ b/crates/compiler-2/README.md @@ -5,6 +5,13 @@ WebAssembly binary. ## Architecture -The compiler is split into 4 +The Rune compiler is base around [Salsa][salsa], a library for incremental +computation. This lets us phrase the compilation process as a series of queries +(essentially, pure functions) which can be aggressively cached based on +dependency analysis. -- [`crate::parse`] - parser +- [`parse`] - Runefile parsing +- [`lowering`] - Convert a Runefile's AST into a high-level intermediate + representation that is more amenable to analysis + +[salsa]: https://github.com/salsa-rs/salsa diff --git a/crates/compiler-2/src/diagnostics/mod.rs b/crates/compiler-2/src/diagnostics/mod.rs new file mode 100644 index 0000000000..058e9c7b79 --- /dev/null +++ b/crates/compiler-2/src/diagnostics/mod.rs @@ -0,0 +1,115 @@ +use im::Vector; + +use crate::Text; + +pub struct DiagnosticMetadata { + pub title: Text, + pub code: Option, + pub description: Option, +} + +impl DiagnosticMetadata { + pub fn new(title: impl Into) -> Self { + DiagnosticMetadata { + title: title.into(), + code: None, + description: None, + } + } + + pub fn with_code(self, code: impl Into) -> Self { + DiagnosticMetadata { + code: Some(code.into()), + ..self + } + } + + pub fn with_description(self, description: impl Into) -> Self { + DiagnosticMetadata { + description: Some(description.into()), + ..self + } + } +} + +/// A severity level for diagnostic messages. +/// +/// These are ordered in the following way: +/// +/// ```rust +/// use hotg_rune_compiler_2::diagnostics::Severity; +/// +/// assert!(Severity::Bug > Severity::Error); +/// assert!(Severity::Error > Severity::Warning); +/// assert!(Severity::Warning > Severity::Note); +/// assert!(Severity::Note > Severity::Help); +/// ``` +#[derive( + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Debug, + serde::Serialize, + serde::Deserialize, +)] +pub enum Severity { + /// A help message. + Help, + /// A note. + Note, + /// A warning. + Warning, + /// An error. + Error, + /// An unexpected bug. + Bug, +} + +/// A collection of [`Diagnostic`]s. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +#[serde(transparent)] +pub struct Diagnostics(Vec); + +/// Something that can be used to create a [`Diagnostic`]. +pub trait AsDiagnostic: std::error::Error { + fn meta() -> DiagnosticMetadata; + fn as_diagnostic(&self) -> Diagnostic; +} + +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct Diagnostic { + pub severity: Severity, + pub message: Text, + pub primary_label: Option FromIterator for Vector { + fn from_iter>(iter: T) -> Self { + Vector(iter.into_iter().collect()) + } +} + +#[derive( + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, + schemars::JsonSchema, +)] +#[repr(transparent)] +#[serde(from = "BTreeMap", into = "BTreeMap")] +#[serde(bound( + serialize = "K: Clone + serde::Serialize + Ord, V: Clone + serde::Serialize + Ord", +))] +pub struct OrdMap(Arc>); + +impl Clone for OrdMap { + fn clone(&self) -> Self { + OrdMap(Arc::clone(&self.0)) + } +} + +impl Default for OrdMap { + fn default() -> Self { + OrdMap::from(BTreeMap::new()) + } +} + +impl From> for OrdMap { + fn from(v: BTreeMap) -> Self { + OrdMap(Arc::new(v)) + } +} + +impl From> for BTreeMap { + fn from(v: OrdMap) -> Self { + BTreeMap::clone(&v.0) + } +} + +impl<'a, K: Ord, V> IntoIterator for &'a OrdMap { + type Item = (&'a K, &'a V); + + type IntoIter = + <&'a std::collections::BTreeMap as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl FromIterator<(K, V)> for OrdMap { + fn from_iter>(iter: T) -> Self { + let map: BTreeMap = iter.into_iter().collect(); + map.into() + } +} diff --git a/crates/compiler-2/src/lib.rs b/crates/compiler-2/src/lib.rs index d93769f7b7..04a46577cf 100644 --- a/crates/compiler-2/src/lib.rs +++ b/crates/compiler-2/src/lib.rs @@ -7,12 +7,12 @@ extern crate pretty_assertions; pub mod codegen; mod config; mod filesystem; +pub mod im; pub mod parse; -mod text; pub mod type_check; pub use crate::{ config::{BuildConfig, Environment, EnvironmentStorage}, filesystem::{FileSystem, ReadError}, - text::Text, + im::Text, }; diff --git a/crates/compiler-2/src/parse/query.rs b/crates/compiler-2/src/parse/query.rs index a34504c197..4fde2df4a2 100644 --- a/crates/compiler-2/src/parse/query.rs +++ b/crates/compiler-2/src/parse/query.rs @@ -1,9 +1,9 @@ -use std::{error::Error, sync::Arc}; +use std::{collections::BTreeMap, error::Error, sync::Arc}; -use im::{OrdMap, Vector}; use uriparse::{URIBuilder, URIError, URI}; use crate::{ + im::{OrdMap, Vector}, parse::{ Document, DocumentV1, ItemType, ModelStage, NotFound, ParseFailed, Path, ProcBlockStage, ResourceDeclaration, Stage, WrongItemType, @@ -22,10 +22,9 @@ use crate::{ /// ```rust /// use hotg_rune_compiler_2::{ /// parse::{Frontend, FrontendStorage}, -/// EnvironmentStorage, FileSystem, ReadError, parse::Path, +/// EnvironmentStorage, FileSystem, ReadError, parse::Path, im::Vector, /// }; /// use uriparse::URI; -/// # use im::Vector; /// /// // First, you need to create a database which can hold Salsa's state /// @@ -101,6 +100,9 @@ pub trait Frontend: Environment + FileSystem { #[salsa::dependencies] fn model_file(&self, name: Text) -> Result, Arc>; + + #[salsa::dependencies] + fn model_files(&self) -> Result>, Arc>; } #[tracing::instrument(skip(src), err)] @@ -152,7 +154,7 @@ fn proc_blocks( ) -> Result>, Arc> { let doc = db.parse()?; - let mut proc_blocks = OrdMap::new(); + let mut proc_blocks = BTreeMap::default(); for (name, stage) in &doc.pipeline { if let Stage::ProcBlock(_) = stage { @@ -161,7 +163,7 @@ fn proc_blocks( } } - Ok(proc_blocks) + Ok(proc_blocks.into()) } #[tracing::instrument(skip(db), err)] @@ -236,6 +238,24 @@ fn model_file( } } +#[tracing::instrument(skip(db))] +fn model_files( + db: &dyn Frontend, +) -> Result>, Arc> { + let doc = db.parse()?; + + let mut models = BTreeMap::default(); + + for (name, stage) in &doc.pipeline { + if let Stage::Model(_) = stage { + let binary = db.model_file(name.into())?; + models.insert(Text::from(name), binary); + } + } + + Ok(models.into()) +} + fn read(db: &dyn Frontend, path: &Path) -> Result, Arc> { file_uri(db, path) .map_err(|e| Arc::new(e) as Arc) diff --git a/crates/compiler-2/src/text.rs b/crates/compiler-2/src/text.rs deleted file mode 100644 index ecb577b577..0000000000 --- a/crates/compiler-2/src/text.rs +++ /dev/null @@ -1,92 +0,0 @@ -use std::{ - borrow::{Borrow, Cow}, - fmt::{self, Display, Formatter}, - sync::Arc, -}; - -/// A reference-counted string. -#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct Text(Arc); - -impl Text { - pub fn new(s: impl Into>) -> Self { - Text(s.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl Display for Text { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From> for Text { - fn from(s: Arc) -> Self { - Text(s) - } -} - -impl From for Text { - fn from(s: String) -> Self { - Text(s.into()) - } -} - -impl From<&'_ str> for Text { - fn from(s: &'_ str) -> Self { - Text(s.into()) - } -} - -impl From<&'_ String> for Text { - fn from(s: &'_ String) -> Self { - Text(s.as_str().into()) - } -} - -impl std::ops::Deref for Text { - type Target = Arc; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Borrow for Text { - fn borrow(&self) -> &str { - &self.0 - } -} - -impl PartialEq for Text -where - T: PartialEq, -{ - fn eq(&self, other: &T) -> bool { - other == self.as_str() - } -} - -impl<'de> serde::Deserialize<'de> for Text { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let s = Cow::<'de, str>::deserialize(deserializer)?; - Ok(Text::new(s)) - } -} - -impl serde::Serialize for Text { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.as_str().serialize(serializer) - } -} diff --git a/crates/compiler-2/src/type_check/types.rs b/crates/compiler-2/src/type_check/types.rs index be8dfe591c..d7435e109b 100644 --- a/crates/compiler-2/src/type_check/types.rs +++ b/crates/compiler-2/src/type_check/types.rs @@ -1,6 +1,6 @@ use std::num::NonZeroUsize; -use im::Vector; +use crate::im::Vector; #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, diff --git a/examples/sine/Runefile.yml b/examples/sine/Runefile.yml index ff1f988c62..5bcf59bbe8 100644 --- a/examples/sine/Runefile.yml +++ b/examples/sine/Runefile.yml @@ -11,7 +11,7 @@ pipeline: - 1 - 1 mod360: - proc-block: "hotg-ai/proc-blocks@v0.11.3#modulo" + proc-block: "wapm://hotg-ai/modulo/v0.11.3" inputs: - rand outputs: From 275e4a0f5e013bdcf98f6d33d0c9837d9b495c3a Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 29 Apr 2022 10:05:36 +0800 Subject: [PATCH 25/84] Implemented the Runefile patching --- crates/compiler-2/src/codegen/query.rs | 150 +++++++++++++++++++++++-- crates/compiler-2/src/parse/yaml.rs | 9 ++ 2 files changed, 150 insertions(+), 9 deletions(-) diff --git a/crates/compiler-2/src/codegen/query.rs b/crates/compiler-2/src/codegen/query.rs index 237790b089..1b500a5aae 100644 --- a/crates/compiler-2/src/codegen/query.rs +++ b/crates/compiler-2/src/codegen/query.rs @@ -4,7 +4,14 @@ use std::{ sync::Arc, }; -use crate::im::{OrdMap, Vector}; +use crate::{ + im::{OrdMap, Vector}, + parse::{ + Argument, Path, ResourceDeclaration, ResourceName, ResourceOrString, + Stage, + }, +}; +use indexmap::IndexMap; use zip::{result::ZipResult, write::FileOptions, ZipWriter}; use crate::{ @@ -15,21 +22,98 @@ use crate::{ #[salsa::query_group(CodegenStorage)] pub trait Codegen: Frontend { /// Create a self-contained [`DocumentV1`] with all resources resolved and - /// - fn self_contained_runefile(&self) -> Arc; + /// pointing to "local" (according to the ZIP archive generated by + /// [`Codegen::rune_archive()`]) files. + #[salsa::dependencies] + fn self_contained_runefile( + &self, + ) -> Result, Arc>; #[salsa::dependencies] fn rune_archive(&self) -> Result, Arc>; } #[tracing::instrument(skip(db))] -fn self_contained_runefile(db: &dyn Codegen) -> Arc { - db.parse().unwrap() +fn self_contained_runefile( + db: &dyn Codegen, +) -> Result, Arc> { + let mut doc = db.parse()?; + + let d = Arc::make_mut(&mut doc); + patch_arguments(db, &mut d.pipeline)?; + patch_paths(&mut d.pipeline)?; + patch_resources(&mut d.resources)?; + + Ok(doc) +} + +#[tracing::instrument(skip(db, stages))] +fn patch_arguments( + db: &dyn Codegen, + stages: &mut IndexMap, +) -> Result<(), Arc> { + for (stage_name, stage) in stages { + for (arg_name, arg_value) in stage.args_mut() { + if let Argument(ResourceOrString::Resource(ResourceName(res))) = + arg_value + { + let value = db.resource_value(res.as_str().into())?; + tracing::debug!( + stage=%stage_name, + arg=%arg_name, + value_len=value.len(), + "Patched an argument", + ); + let s = std::str::from_utf8(&value) + .map_err(|e| Arc::new(e) as Arc)?; + *arg_value = Argument(ResourceOrString::String(s.to_string())); + } + } + } + + Ok(()) +} + +#[tracing::instrument(skip(stages))] +fn patch_paths( + stages: &mut IndexMap, +) -> Result<(), Arc> { + for (name, stage) in stages { + match stage { + Stage::Model(m) => { + let path = Path::FileSystem(format!("models/{name}")); + tracing::debug!(new=%path, old=%m.model, "Patching model path"); + m.model = path; + }, + Stage::ProcBlock(p) => { + let path = Path::FileSystem(format!("proc_blocks/{name}")); + tracing::debug!(new=%path, old=%p.proc_block, "Patching proc-block path"); + p.proc_block = path; + }, + Stage::Capability(_) | Stage::Out(_) => {}, + } + } + + Ok(()) +} + +#[tracing::instrument(skip(resources))] +fn patch_resources( + resources: &mut IndexMap, +) -> Result<(), Arc> { + for (name, decl) in resources { + decl.inline = None; + let path = format!("resources/{name}"); + tracing::debug!(?path, "Patched resource"); + decl.path = Some(path); + } + + Ok(()) } #[tracing::instrument(skip(db))] fn rune_archive(db: &dyn Codegen) -> Result, Arc> { - let runefile = db.self_contained_runefile(); + let runefile = db.self_contained_runefile()?; let runefile = serde_yaml::to_string(&*runefile) .expect("Serializing to YAML should never fail"); let proc_blocks = db.proc_blocks()?; @@ -78,6 +162,7 @@ fn write_to_directory( data: &[u8], ) -> ZipResult<()> { let path = format!("{directory}/{name}"); + tracing::debug!(%directory, %path, bytes = %data.len(), "Writing to file"); writer.start_file(&path, FileOptions::default())?; writer.write_all(data)?; Ok(()) @@ -85,7 +170,7 @@ fn write_to_directory( #[cfg(test)] mod tests { - use std::path::Path; + use std::{path::Path}; use tracing_test::traced_test; use uriparse::{Scheme, URI}; use zip::ZipArchive; @@ -139,13 +224,60 @@ mod tests { let zune = db.rune_archive().unwrap(); - let reader = ZipArchive::new(Cursor::new(&*zune)).unwrap(); + let mut reader = ZipArchive::new(Cursor::new(&*zune)).unwrap(); - let mut entries: Vec<_> = reader.file_names().collect(); + let mut entries: Vec<_> = + reader.file_names().map(String::from).collect(); entries.sort(); assert_eq!( entries, &["Runefile.yml", "models/sine", "proc_blocks/mod360",] ); + + let expected = r#" + version: 1 + image: runicos/base + pipeline: + rand: + capability: RAW + outputs: + - type: F32 + dimensions: + - 1 + - 1 + args: + length: "4" + mod360: + proc-block: proc_blocks/mod360 + inputs: + - rand + outputs: + - type: F32 + dimensions: + - 1 + - 1 + args: + modulus: "360" + sine: + model: models/sine + inputs: + - mod360 + outputs: + - type: F32 + dimensions: + - 1 + - 1 + serial: + out: serial + inputs: + - sine + resources: {}"#; + + let f = reader.by_name("Runefile.yml").unwrap(); + let actual: serde_yaml::Value = serde_yaml::from_reader(f).unwrap(); + let expected: serde_yaml::Value = + serde_yaml::from_str(expected).unwrap(); + + assert_eq!(actual, expected); } } diff --git a/crates/compiler-2/src/parse/yaml.rs b/crates/compiler-2/src/parse/yaml.rs index 67c4db5da3..fb902e54c7 100644 --- a/crates/compiler-2/src/parse/yaml.rs +++ b/crates/compiler-2/src/parse/yaml.rs @@ -385,6 +385,15 @@ impl Stage { Stage::Out(out) => &out.args, } } + + pub(crate) fn args_mut(&mut self) -> &mut IndexMap { + match self { + Stage::Model(m) => &mut m.args, + Stage::ProcBlock(p) => &mut p.args, + Stage::Capability(c) => &mut c.args, + Stage::Out(out) => &mut out.args, + } + } } /// Something that could be either a reference to a resource (`$resource`) From c2deb866953a2f993974a8482edf55a8b81c1feb Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 29 Apr 2022 10:10:21 +0800 Subject: [PATCH 26/84] Deleted the old compiler implementation --- Cargo.lock | 247 +--- crates/compiler-2/Cargo.toml | 29 - crates/compiler-2/README.md | 18 - crates/compiler-2/runefile-schema.json | 263 ---- crates/compiler-2/src/codegen/mod.rs | 3 - crates/compiler-2/src/lib.rs | 18 - crates/compiler-2/src/parse/mod.rs | 82 -- crates/compiler-2/src/parse/yaml.rs | 1190 ---------------- crates/compiler-2/src/type_check/mod.rs | 5 - crates/compiler/Cargo.toml | 47 +- crates/compiler/error-codes.md | 9 - crates/compiler/examples/extensions.rs | 150 -- crates/compiler/runefile-schema.json | 191 ++- crates/compiler/src/build_context.rs | 158 --- .../src/codegen/compile_generated_project.rs | 2 - crates/compiler/src/codegen/components.rs | 193 --- .../src/codegen/generate_cargo_config.rs | 110 -- .../src/codegen/generate_cargo_toml.rs | 387 ------ .../compiler/src/codegen/generate_lib_rs.rs | 1215 ----------------- .../src/codegen/generate_model_files.rs | 17 - .../src/codegen/generate_resource_section.rs | 67 - .../codegen/generate_rune_graph_section.rs | 217 --- .../codegen/generate_rust_toolchain_toml.rs | 13 - .../src/codegen/generate_version_section.rs | 19 - crates/compiler/src/codegen/inputs.rs | 57 - crates/compiler/src/codegen/mod.rs | 150 +- .../src/codegen/query.rs | 0 crates/compiler/src/compile/cargo_build.rs | 89 -- crates/compiler/src/compile/components.rs | 70 - crates/compiler/src/compile/mod.rs | 34 - .../src/compile/write_project_to_disk.rs | 26 - crates/{compiler-2 => compiler}/src/config.rs | 0 crates/compiler/src/diagnostics.rs | 71 - .../src/filesystem.rs | 0 crates/compiler/src/hooks.rs | 175 --- crates/{compiler-2 => compiler}/src/im.rs | 0 crates/compiler/src/inputs.rs | 19 - crates/compiler/src/lib.rs | 46 +- crates/compiler/src/lowering/components.rs | 377 ----- .../compiler/src/lowering/load_model_data.rs | 33 - .../src/lowering/load_resource_data.rs | 122 -- crates/compiler/src/lowering/mod.rs | 51 - crates/compiler/src/lowering/query.rs | 232 ---- .../compiler/src/lowering/register_names.rs | 19 - .../src/lowering/register_resources.rs | 167 --- .../compiler/src/lowering/register_stages.rs | 489 ------- .../compiler/src/lowering/register_tensors.rs | 320 ----- .../compiler/src/lowering/update_nametable.rs | 78 -- crates/compiler/src/macros.rs | 29 - crates/compiler/src/parse/mod.rs | 92 +- .../src/parse/query.rs | 2 +- crates/compiler/src/parse/yaml.rs | 418 +++--- crates/compiler/src/phases.rs | 313 ----- crates/compiler/src/serialize.rs | 63 - crates/compiler/src/toolchain.rs | 27 - .../src/type_check/check_for_loops.rs | 114 -- crates/compiler/src/type_check/components.rs | 1 - crates/compiler/src/type_check/mod.rs | 19 +- .../src/type_check/model_args_are_consumed.rs | 39 - .../src/type_check/types.rs | 0 crates/compiler/tests/existing_runefiles.rs | 184 --- 61 files changed, 374 insertions(+), 8202 deletions(-) delete mode 100644 crates/compiler-2/Cargo.toml delete mode 100644 crates/compiler-2/README.md delete mode 100644 crates/compiler-2/runefile-schema.json delete mode 100644 crates/compiler-2/src/codegen/mod.rs delete mode 100644 crates/compiler-2/src/lib.rs delete mode 100644 crates/compiler-2/src/parse/mod.rs delete mode 100644 crates/compiler-2/src/parse/yaml.rs delete mode 100644 crates/compiler-2/src/type_check/mod.rs delete mode 100644 crates/compiler/error-codes.md delete mode 100644 crates/compiler/examples/extensions.rs delete mode 100644 crates/compiler/src/build_context.rs delete mode 100644 crates/compiler/src/codegen/compile_generated_project.rs delete mode 100644 crates/compiler/src/codegen/components.rs delete mode 100644 crates/compiler/src/codegen/generate_cargo_config.rs delete mode 100644 crates/compiler/src/codegen/generate_cargo_toml.rs delete mode 100644 crates/compiler/src/codegen/generate_lib_rs.rs delete mode 100644 crates/compiler/src/codegen/generate_model_files.rs delete mode 100644 crates/compiler/src/codegen/generate_resource_section.rs delete mode 100644 crates/compiler/src/codegen/generate_rune_graph_section.rs delete mode 100644 crates/compiler/src/codegen/generate_rust_toolchain_toml.rs delete mode 100644 crates/compiler/src/codegen/generate_version_section.rs delete mode 100644 crates/compiler/src/codegen/inputs.rs rename crates/{compiler-2 => compiler}/src/codegen/query.rs (100%) delete mode 100644 crates/compiler/src/compile/cargo_build.rs delete mode 100644 crates/compiler/src/compile/components.rs delete mode 100644 crates/compiler/src/compile/mod.rs delete mode 100644 crates/compiler/src/compile/write_project_to_disk.rs rename crates/{compiler-2 => compiler}/src/config.rs (100%) delete mode 100644 crates/compiler/src/diagnostics.rs rename crates/{compiler-2 => compiler}/src/filesystem.rs (100%) delete mode 100644 crates/compiler/src/hooks.rs rename crates/{compiler-2 => compiler}/src/im.rs (100%) delete mode 100644 crates/compiler/src/inputs.rs delete mode 100644 crates/compiler/src/lowering/components.rs delete mode 100644 crates/compiler/src/lowering/load_model_data.rs delete mode 100644 crates/compiler/src/lowering/load_resource_data.rs delete mode 100644 crates/compiler/src/lowering/mod.rs delete mode 100644 crates/compiler/src/lowering/query.rs delete mode 100644 crates/compiler/src/lowering/register_names.rs delete mode 100644 crates/compiler/src/lowering/register_resources.rs delete mode 100644 crates/compiler/src/lowering/register_stages.rs delete mode 100644 crates/compiler/src/lowering/register_tensors.rs delete mode 100644 crates/compiler/src/lowering/update_nametable.rs delete mode 100644 crates/compiler/src/macros.rs rename crates/{compiler-2 => compiler}/src/parse/query.rs (99%) delete mode 100644 crates/compiler/src/phases.rs delete mode 100644 crates/compiler/src/serialize.rs delete mode 100644 crates/compiler/src/toolchain.rs delete mode 100644 crates/compiler/src/type_check/check_for_loops.rs delete mode 100644 crates/compiler/src/type_check/components.rs delete mode 100644 crates/compiler/src/type_check/model_args_are_consumed.rs rename crates/{compiler-2 => compiler}/src/type_check/types.rs (100%) delete mode 100644 crates/compiler/tests/existing_runefiles.rs diff --git a/Cargo.lock b/Cargo.lock index 895a2b0ab6..bc7bbbe327 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,12 +85,6 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "atomic_refcell" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" - [[package]] name = "atty" version = "0.2.14" @@ -188,15 +182,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "block-buffer" version = "0.10.2" @@ -336,9 +321,6 @@ name = "bytes" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" -dependencies = [ - "serde", -] [[package]] name = "bzip2" @@ -420,7 +402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e3973b165dc0f435831a9e426de67e894de532754ff7a3f307c03ee5dec7dc" dependencies = [ "clap", - "heck 0.3.3", + "heck", "indexmap", "log", "proc-macro2", @@ -514,23 +496,12 @@ dependencies = [ "cc", ] -[[package]] -name = "codespan" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3362992a0d9f1dd7c3d0e89e0ab2bb540b7a95fea8cd798090e758fda2899b5e" -dependencies = [ - "codespan-reporting", - "serde", -] - [[package]] name = "codespan-reporting" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ - "serde", "termcolor", "unicode-width", ] @@ -916,12 +887,6 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" -[[package]] -name = "downcast-rs" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" - [[package]] name = "dyn-clone" version = "1.0.5" @@ -1003,15 +968,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "erased-serde" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad132dd8d0d0b546348d7d86cb3191aad14b34e5f979781fc005c80d4ac67ffd" -dependencies = [ - "serde", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1300,12 +1256,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "heck" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" - [[package]] name = "hermit-abi" version = "0.1.19" @@ -1365,46 +1315,10 @@ dependencies = [ [[package]] name = "hotg-rune-compiler" version = "0.11.3" -dependencies = [ - "atomic_refcell", - "bytes", - "cargo_toml", - "codespan", - "codespan-reporting", - "env_logger", - "heck 0.4.0", - "hotg-rune-core", - "hotg-rune-proc-blocks", - "im", - "indexmap", - "indoc", - "jsonschema 0.16.0", - "legion", - "log", - "miette", - "once_cell", - "pretty_assertions 1.2.1", - "proc-macro2", - "quote", - "regex", - "salsa", - "schemars", - "serde", - "serde_json", - "serde_yaml", - "thiserror", - "toml", - "tracing", - "zip 0.5.13", -] - -[[package]] -name = "hotg-rune-compiler-2" -version = "0.0.0" dependencies = [ "indexmap", "insta", - "jsonschema 0.15.2", + "jsonschema", "once_cell", "pretty_assertions 1.2.1", "regex", @@ -1627,21 +1541,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "im" -version = "15.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "111c1983f3c5bb72732df25cddacee9b546d08325fb584b5ebd38148be7b0246" -dependencies = [ - "bitmaps", - "rand_core 0.5.1", - "rand_xoshiro", - "serde", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "image" version = "0.23.14" @@ -1672,15 +1571,6 @@ dependencies = [ "serde", ] -[[package]] -name = "indoc" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7906a9fababaeacb774f72410e497a1d18de916322e33797bb2cd29baa23c9e" -dependencies = [ - "unindent", -] - [[package]] name = "insta" version = "1.14.0" @@ -1796,33 +1686,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "jsonschema" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebd40599e7f1230ce296f73b88c022b98ed66689f97eaa54bbeadc337a2ffa6" -dependencies = [ - "ahash", - "anyhow", - "base64", - "bytecount", - "fancy-regex", - "fraction", - "iso8601", - "itoa 1.0.1", - "lazy_static", - "memchr", - "num-cmp", - "parking_lot 0.12.0", - "percent-encoding", - "regex", - "serde", - "serde_json", - "time 0.3.9", - "url", - "uuid", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -1841,39 +1704,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" -[[package]] -name = "legion" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bfd53bb4690a5ab2bd6d1c683461ee52763afe0d000929743708a256d9d9b1f" -dependencies = [ - "atomic_refcell", - "bit-set", - "downcast-rs", - "erased-serde", - "itertools", - "legion_codegen", - "parking_lot 0.11.2", - "paste", - "scoped-tls-hkt", - "serde", - "smallvec", - "thiserror", - "uuid", -] - -[[package]] -name = "legion_codegen" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad5ad7a361d7b2522010335d95fa73135cb3c6816bef22cc7a5d5861587ae1b" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "thiserror", -] - [[package]] name = "libc" version = "0.2.124" @@ -2020,29 +1850,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "miette" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8218047465aa2a0aa5ceca2b5065ae27b54f744c16bf9272fdbc39917362785f" -dependencies = [ - "miette-derive", - "once_cell", - "thiserror", - "unicode-width", -] - -[[package]] -name = "miette-derive" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d56e20719ef4fc7fd02960f72a2451a724e3c348627a941e2f37dda37b247" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "mime" version = "0.3.16" @@ -2419,12 +2226,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "paste" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" - [[package]] name = "pbkdf2" version = "0.10.1" @@ -2705,12 +2506,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" - [[package]] name = "rand_core" version = "0.6.3" @@ -2720,15 +2515,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_xoshiro" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fcdd2e881d02f1d9390ae47ad8e5696a9e4be7b547a1da2afbc61973217004" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rayon" version = "1.5.2" @@ -2985,7 +2771,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd3904a4ba0a9d0211816177fd34b04c7095443f8cdacd11175064fe541c8fe2" dependencies = [ - "heck 0.3.3", + "heck", "proc-macro2", "quote", "syn", @@ -3035,12 +2821,6 @@ dependencies = [ "syn", ] -[[package]] -name = "scoped-tls-hkt" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e9d7eaddb227e8fbaaa71136ae0e1e913ca159b86c7da82f3e8f0044ad3a63" - [[package]] name = "scoped_threadpool" version = "0.1.9" @@ -3254,16 +3034,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e24979f63a11545f5f2c60141afe249d4f19f84581ea2138065e400941d83d3" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "slab" version = "0.4.6" @@ -3315,7 +3085,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ - "heck 0.3.3", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -3337,7 +3107,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "339f799d8b549e3744c7ac7feb216383e4005d94bdb22561b3ab8f3b808ae9fb" dependencies = [ - "heck 0.3.3", + "heck", "proc-macro2", "quote", "syn", @@ -3720,12 +3490,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" -[[package]] -name = "unindent" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514672a55d7380da379785a4d70ca8386c8883ff7eaae877be4d2081cebe73d8" - [[package]] name = "uriparse" version = "0.6.4" @@ -3755,7 +3519,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom", - "serde", ] [[package]] diff --git a/crates/compiler-2/Cargo.toml b/crates/compiler-2/Cargo.toml deleted file mode 100644 index 054aa22224..0000000000 --- a/crates/compiler-2/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "hotg-rune-compiler-2" -version = "0.0.0" -edition = "2021" -publish = false -license = "MIT OR Apache-2.0" -readme = "README.md" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -indexmap = { version = "1.8.1", features = ["serde-1"] } -once_cell = "1.10.0" -regex = "1.5.5" -salsa = "0.16.1" -schemars = { version = "0.8.8", features = ["indexmap"] } -serde = "1.0.136" -serde_json = "1.0.79" -serde_yaml = "0.8.23" -thiserror = "1.0.30" -tracing = "0.1.34" -uriparse = "0.6.4" -zip = "0.6.2" - -[dev-dependencies] -insta = "1.14.0" -jsonschema = "0.15.2" -pretty_assertions = "1.2.1" -tracing-test = "0.2.1" diff --git a/crates/compiler-2/README.md b/crates/compiler-2/README.md deleted file mode 100644 index 1110c991f0..0000000000 --- a/crates/compiler-2/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# The Rune Compiler - -A compiler that compiles your data processing pipeline into a portable -WebAssembly binary. - -## Architecture - -The Rune compiler is base around [Salsa][salsa], a library for incremental -computation. This lets us phrase the compilation process as a series of queries -(essentially, pure functions) which can be aggressively cached based on -dependency analysis. - -These series of queries are broken up into a couple submodules, - -- [`parse`] - Parse a Runefile written in the YAML format -- [`codegen`] - Generate the final Rune binary - -[salsa]: https://github.com/salsa-rs/salsa diff --git a/crates/compiler-2/runefile-schema.json b/crates/compiler-2/runefile-schema.json deleted file mode 100644 index 522d39b6b9..0000000000 --- a/crates/compiler-2/runefile-schema.json +++ /dev/null @@ -1,263 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "anyOf": [ - { - "$ref": "#/definitions/DocumentV1" - } - ], - "definitions": { - "Argument": { - "anyOf": [ - { - "$ref": "#/definitions/ResourceName" - }, - { - "type": "string" - }, - { - "type": "number" - } - ], - "description": "Something that could be either a reference to a resource (`$resource`) or a plain string (`./path`)." - }, - "CapabilityStage": { - "description": "A stage which reads inputs from the runtime.", - "properties": { - "args": { - "additionalProperties": { - "$ref": "#/definitions/Argument" - }, - "type": "object" - }, - "capability": { - "description": "What type of capability to use (\"IMAGE\", \"SOUND\", etc.).", - "type": "string" - }, - "outputs": { - "items": { - "$ref": "#/definitions/Type" - }, - "type": "array" - } - }, - "required": [ - "capability" - ], - "type": "object" - }, - "DocumentV1": { - "description": "Version 1 of the `Runefile.yml` format.", - "properties": { - "image": { - "description": "The base image that defines the interface between a Rune and its runtime.\n\nThis should always be `\"runicos/base\"`.", - "type": "string" - }, - "pipeline": { - "additionalProperties": { - "$ref": "#/definitions/Stage" - }, - "description": "The various stages in the Runefile's pipeline.", - "type": "object" - }, - "resources": { - "additionalProperties": { - "$ref": "#/definitions/ResourceDeclaration" - }, - "default": {}, - "description": "Any resources that can be accessed by pipeline stages.", - "type": "object" - }, - "version": { - "description": "The version number. Must always be `\"1\"`.", - "format": "uint", - "maximum": 1.0, - "minimum": 1.0, - "type": "integer" - } - }, - "required": [ - "image", - "pipeline", - "version" - ], - "type": "object" - }, - "Input": { - "description": "\nThe name of a tensor.\n\nTypically something like \"stage\", or \"stage.2\" if the stage has multiple outputs.\n", - "format": "string", - "pattern": "^(?P[a-zA-Z_][\\w-]*)(?:\\.(?P\\d+))?$", - "type": "string" - }, - "ModelStage": { - "description": "A ML model which will be executed by the runtime.", - "properties": { - "args": { - "additionalProperties": { - "$ref": "#/definitions/Argument" - }, - "type": "object" - }, - "inputs": { - "description": "Tensors to use as input to this model.", - "items": { - "$ref": "#/definitions/Input" - }, - "type": "array" - }, - "model": { - "description": "The model to use, or a resource which specifies the model to use.", - "type": "string" - }, - "outputs": { - "description": "The tensors that this model outputs.", - "items": { - "$ref": "#/definitions/Type" - }, - "type": "array" - } - }, - "required": [ - "model" - ], - "type": "object" - }, - "OutStage": { - "description": "A stage which passes outputs back to the runtime.", - "properties": { - "args": { - "additionalProperties": { - "$ref": "#/definitions/Argument" - }, - "type": "object" - }, - "inputs": { - "items": { - "$ref": "#/definitions/Input" - }, - "type": "array" - }, - "out": { - "description": "The type of output (e.g. \"SERIAL\").", - "type": "string" - } - }, - "required": [ - "out" - ], - "type": "object" - }, - "ProcBlockStage": { - "description": "A stage which executes a procedural block.", - "properties": { - "args": { - "additionalProperties": { - "$ref": "#/definitions/Argument" - }, - "type": "object" - }, - "inputs": { - "items": { - "$ref": "#/definitions/Input" - }, - "type": "array" - }, - "outputs": { - "items": { - "$ref": "#/definitions/Type" - }, - "type": "array" - }, - "proc-block": { - "description": "A [`Path`] that Rune can use to locate the proc block.", - "type": "string" - } - }, - "required": [ - "proc-block" - ], - "type": "object" - }, - "ResourceDeclaration": { - "additionalProperties": false, - "description": "The declaration for a resource, typically something like a wordlist or environment variable.", - "properties": { - "inline": { - "description": "A resource who's default value is specified inline.", - "type": [ - "string", - "null" - ] - }, - "path": { - "description": "A resource who's default value is meant to be loaded from a file.", - "type": [ - "string", - "null" - ] - }, - "type": { - "allOf": [ - { - "$ref": "#/definitions/ResourceType" - } - ], - "default": "string" - } - }, - "type": "object" - }, - "ResourceName": { - "description": "\nA reference to some [`ResourceDeclaration`]. It typically looks like\n`$RESOURCE_NAME`.\n", - "format": "string", - "pattern": "^\\$[_a-zA-Z][_a-zA-Z0-9]*$", - "type": "string" - }, - "ResourceType": { - "description": "How the resource should be treated inside the Rune.", - "enum": [ - "string", - "binary" - ], - "type": "string" - }, - "Stage": { - "anyOf": [ - { - "$ref": "#/definitions/ModelStage" - }, - { - "$ref": "#/definitions/ProcBlockStage" - }, - { - "$ref": "#/definitions/CapabilityStage" - }, - { - "$ref": "#/definitions/OutStage" - } - ], - "description": "A stage in the Rune's pipeline." - }, - "Type": { - "description": "The element type and dimensions for a particular tensor.", - "properties": { - "dimensions": { - "items": { - "format": "uint", - "minimum": 0.0, - "type": "integer" - }, - "type": "array" - }, - "type": { - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - } - }, - "description": "The top level Runefile type.", - "title": "Document" -} \ No newline at end of file diff --git a/crates/compiler-2/src/codegen/mod.rs b/crates/compiler-2/src/codegen/mod.rs deleted file mode 100644 index 68c159aa84..0000000000 --- a/crates/compiler-2/src/codegen/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod query; - -pub use self::query::{Codegen, CodegenStorage}; diff --git a/crates/compiler-2/src/lib.rs b/crates/compiler-2/src/lib.rs deleted file mode 100644 index 04a46577cf..0000000000 --- a/crates/compiler-2/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -#![doc= include_str!("../README.md")] - -#[cfg(test)] -#[macro_use] -extern crate pretty_assertions; - -pub mod codegen; -mod config; -mod filesystem; -pub mod im; -pub mod parse; -pub mod type_check; - -pub use crate::{ - config::{BuildConfig, Environment, EnvironmentStorage}, - filesystem::{FileSystem, ReadError}, - im::Text, -}; diff --git a/crates/compiler-2/src/parse/mod.rs b/crates/compiler-2/src/parse/mod.rs deleted file mode 100644 index 24191fc66e..0000000000 --- a/crates/compiler-2/src/parse/mod.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! The YAML frontend for the Rune compiler. -//! -//! You are probably here for either the [`Frontend`] trait or the [`Document`] -//! type. - -mod query; -mod yaml; - -use std::sync::Arc; - -use crate::Text; - -pub use self::{ - query::{Frontend, FrontendStorage}, - yaml::*, -}; - -#[derive(Debug, Clone, thiserror::Error)] -#[error("Unable to parse the Runefile")] -pub struct ParseFailed { - #[from] - pub error: Arc, -} - -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -#[serde(rename_all = "kebab-case")] -pub enum ItemType { - Input, - Model, - ProcBlock, - Output, - Resource, -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Hash, - thiserror::Error, - serde::Serialize, - serde::Deserialize, -)] -#[error("There is no model called \"{}\"", name)] -#[serde(rename_all = "kebab-case")] -pub struct NotFound { - pub item_type: ItemType, - pub name: Text, -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Hash, - thiserror::Error, - serde::Serialize, - serde::Deserialize, -)] -#[error( - "Expected \"{}\" to be a {:?}, but it is actually a {:?}", - name, - expected, - actual -)] -#[serde(rename_all = "kebab-case")] -pub struct WrongItemType { - pub expected: ItemType, - pub actual: ItemType, - pub name: Text, -} diff --git a/crates/compiler-2/src/parse/yaml.rs b/crates/compiler-2/src/parse/yaml.rs deleted file mode 100644 index fb902e54c7..0000000000 --- a/crates/compiler-2/src/parse/yaml.rs +++ /dev/null @@ -1,1190 +0,0 @@ -//! Definitions for the Runefile's YAML format. - -use std::{ - borrow::Cow, - fmt::{self, Display, Formatter}, - ops::Deref, - str::FromStr, -}; - -use indexmap::IndexMap; -use once_cell::sync::Lazy; -use regex::Regex; -use schemars::{ - gen::SchemaGenerator, - schema::{ - InstanceType, Metadata, Schema, SchemaObject, SubschemaValidation, - }, - JsonSchema, -}; -use serde::{ - de::{Deserialize, Deserializer, Error as _}, - ser::{Serialize, Serializer}, -}; -use uriparse::{URIError, URI}; - -static RESOURCE_NAME_PATTERN: Lazy = - Lazy::new(|| Regex::new(r"^\$[_a-zA-Z][_a-zA-Z0-9]*$").unwrap()); - -/// The top level Runefile type. -#[derive(Debug, Clone, PartialEq, JsonSchema)] -#[schemars(untagged)] -pub enum Document { - V1(DocumentV1), -} - -impl Document { - pub fn to_v1(self) -> DocumentV1 { - match self { - Document::V1(d) => d, - } - } -} - -impl From for Document { - fn from(v1: DocumentV1) -> Self { - Document::V1(v1) - } -} - -mod document_serde { - use serde::de::Unexpected; - use serde_yaml::Value; - - use super::*; - - #[derive(serde::Serialize, serde::Deserialize)] - struct Repr { - version: usize, - #[serde(flatten)] - inner: T, - } - - impl Repr { - fn new(version: usize, inner: T) -> Self { - Repr { version, inner } - } - } - - impl Serialize for Document { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - Document::V1(v1) => Repr::new(1, v1).serialize(serializer), - } - } - } - - impl<'de> Deserialize<'de> for Document { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = Value::deserialize(deserializer)?; - let version_key = Value::from("version"); - let version = value - .as_mapping() - .and_then(|m| m.get(&version_key)) - .and_then(|v| v.as_u64()); - - match version { - Some(1) => { - let v1: DocumentV1 = serde_yaml::from_value(value) - .map_err(D::Error::custom)?; - Ok(Document::V1(v1)) - }, - Some(other) => Err(D::Error::invalid_value( - Unexpected::Unsigned(other), - &"version to be 1", - )), - None => Err(D::Error::missing_field("version")), - } - } - } -} - -macro_rules! impl_json_schema_via_regex { - ($ty:ty, $pattern:expr, $docs:literal) => { - impl JsonSchema for $ty { - fn schema_name() -> String { - String::from(stringify!($ty)) - } - - fn json_schema(_: &mut SchemaGenerator) -> Schema { - let mut schema = SchemaObject { - instance_type: Some(InstanceType::String.into()), - format: Some(String::from("string")), - metadata: Some(Box::new(Metadata { - description: Some(String::from($docs)), - ..Default::default() - })), - ..Default::default() - }; - - schema.string().pattern = Some($pattern.to_string()); - - schema.into() - } - } - }; -} - -/// Version 1 of the `Runefile.yml` format. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -pub struct DocumentV1 { - /// The version number. Must always be `"1"`. - #[schemars(required, range(min = 1, max = 1))] - pub version: usize, - /// The base image that defines the interface between a Rune and its - /// runtime. - /// - /// This should always be `"runicos/base"`. - pub image: Image, - /// The various stages in the Runefile's pipeline. - pub pipeline: IndexMap, - /// Any resources that can be accessed by pipeline stages. - #[serde(default)] - pub resources: IndexMap, -} - -impl Document { - pub fn parse(yaml: &str) -> Result { - serde_yaml::from_str(yaml) - } - - pub fn write_as_yaml(&self, writer: W) -> Result<(), serde_yaml::Error> - where - W: std::io::Write, - { - serde_yaml::to_writer(writer, self)?; - Ok(()) - } -} - -impl FromStr for Document { - type Err = serde_yaml::Error; - - fn from_str(s: &str) -> Result { - Document::parse(s) - } -} - -/// A ML model which will be executed by the runtime. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -pub struct ModelStage { - /// The model to use, or a resource which specifies the model to use. - #[schemars(required)] - pub model: Path, - /// Tensors to use as input to this model. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - /// The tensors that this model outputs. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub outputs: Vec, - #[serde(default, skip_serializing_if = "IndexMap::is_empty")] - pub args: IndexMap, -} - -/// A stage which executes a procedural block. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -pub struct ProcBlockStage { - /// A [`Path`] that Rune can use to locate the proc block. - #[serde(rename = "proc-block")] - #[schemars(required)] - pub proc_block: Path, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub outputs: Vec, - #[serde(default, skip_serializing_if = "IndexMap::is_empty")] - pub args: IndexMap, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum Path { - Uri(URI<'static>), - FileSystem(String), -} - -impl Serialize for Path { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - self.to_string().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for Path { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer)?; - - s.parse().map_err(D::Error::custom) - } -} - -impl FromStr for Path { - type Err = URIError; - - fn from_str(s: &str) -> Result { - match URI::try_from(s) { - Ok(u) => Ok(Path::Uri(u.into_owned())), - Err(URIError::NotURI) => Ok(Path::FileSystem(s.to_string())), - Err(e) => Err(e), - } - } -} - -impl Display for Path { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - Path::Uri(u) => u.fmt(f), - Path::FileSystem(p) => p.fmt(f), - } - } -} - -impl JsonSchema for Path { - fn schema_name() -> String { - String::from("Path") - } - - fn json_schema(gen: &mut schemars::gen::SchemaGenerator) -> Schema { - gen.subschema_for::() - } -} - -/// A stage which reads inputs from the runtime. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -pub struct CapabilityStage { - /// What type of capability to use ("IMAGE", "SOUND", etc.). - #[schemars(required)] - pub capability: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub outputs: Vec, - #[serde(default, skip_serializing_if = "IndexMap::is_empty")] - pub args: IndexMap, -} - -/// A stage which passes outputs back to the runtime. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -pub struct OutStage { - /// The type of output (e.g. "SERIAL"). - #[schemars(required)] - pub out: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - #[serde(default, skip_serializing_if = "IndexMap::is_empty")] - pub args: IndexMap, -} - -/// A stage in the Rune's pipeline. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - JsonSchema, -)] -#[serde(untagged, rename_all = "kebab-case")] -pub enum Stage { - Model(ModelStage), - ProcBlock(ProcBlockStage), - Capability(CapabilityStage), - Out(OutStage), -} - -impl Stage { - pub fn inputs(&self) -> &[Input] { - match self { - Stage::Model(ModelStage { inputs, .. }) - | Stage::ProcBlock(ProcBlockStage { inputs, .. }) - | Stage::Out(OutStage { inputs, .. }) => inputs, - Stage::Capability(_) => &[], - } - } - - pub fn inputs_mut(&mut self) -> Option<&mut Vec> { - match self { - Stage::Model(ModelStage { inputs, .. }) - | Stage::ProcBlock(ProcBlockStage { inputs, .. }) - | Stage::Out(OutStage { inputs, .. }) => Some(inputs), - Stage::Capability(_) => None, - } - } - - pub fn output_type(&self) -> Option<&Type> { - match self.output_types() { - [] => None, - [output] => Some(output), - _ => unimplemented!("Multiple outputs aren't supported yet"), - } - } - - pub fn output_types(&self) -> &[Type] { - match self { - Stage::Model(ModelStage { outputs, .. }) - | Stage::ProcBlock(ProcBlockStage { outputs, .. }) - | Stage::Capability(CapabilityStage { outputs, .. }) => outputs, - Stage::Out(OutStage { .. }) => &[], - } - } - - pub fn args(&self) -> &IndexMap { - match self { - Stage::Model(m) => &m.args, - Stage::ProcBlock(p) => &p.args, - Stage::Capability(c) => &c.args, - Stage::Out(out) => &out.args, - } - } - - pub(crate) fn args_mut(&mut self) -> &mut IndexMap { - match self { - Stage::Model(m) => &mut m.args, - Stage::ProcBlock(p) => &mut p.args, - Stage::Capability(c) => &mut c.args, - Stage::Out(out) => &mut out.args, - } - } -} - -/// Something that could be either a reference to a resource (`$resource`) -/// or a plain string (`./path`). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum ResourceOrString { - Resource(ResourceName), - String(String), -} - -impl JsonSchema for ResourceOrString { - fn schema_name() -> std::string::String { - "ResourceOrString".to_owned() - } - - fn json_schema(gen: &mut SchemaGenerator) -> Schema { - let resource_name = gen.subschema_for::(); - let string = gen.subschema_for::(); - - let description = "Something that could be either a reference to a \ - resource (`$resource`) or a plain string \ - (`./path`)."; - - Schema::Object(SchemaObject { - metadata: Some(Box::new(Metadata { - description: Some(description.to_owned()), - ..Default::default() - })), - subschemas: Some(Box::new(SubschemaValidation { - any_of: Some(vec![resource_name, string]), - ..Default::default() - })), - ..Default::default() - }) - } -} - -impl Serialize for ResourceOrString { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - self.to_string().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for ResourceOrString { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct Visitor; - - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = ResourceOrString; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "a number, string, or \"$RESOURCE_NAME\"") - } - - fn visit_u64(self, v: u64) -> Result - where - E: serde::de::Error, - { - Ok(ResourceOrString::String(v.to_string())) - } - - fn visit_i64(self, v: i64) -> Result - where - E: serde::de::Error, - { - Ok(ResourceOrString::String(v.to_string())) - } - - fn visit_f64(self, v: f64) -> Result - where - E: serde::de::Error, - { - Ok(ResourceOrString::String(v.to_string())) - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - let v = v.trim(); - - if !v.starts_with('$') { - return Ok(ResourceOrString::String(v.to_string())); - } - - match ResourceName::from_str(v) { - Ok(name) => Ok(ResourceOrString::Resource(name)), - Err(e) => Err(E::custom(e)), - } - } - - fn visit_seq(self, _: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - Err(A::Error::custom("lists aren't supported")) - } - } - - deserializer.deserialize_any(Visitor) - } -} - -impl Display for ResourceOrString { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - ResourceOrString::String(path) => write!(f, "{}", path), - ResourceOrString::Resource(res) => write!(f, "{}", res), - } - } -} - -impl> From for ResourceOrString { - fn from(s: S) -> Self { - ResourceOrString::String(s.into()) - } -} - -impl From for ResourceOrString { - fn from(name: ResourceName) -> Self { - ResourceOrString::Resource(name) - } -} - -/// A newtype around [`ResourceOrString`] which is used in each stage's `args` -/// dictionary. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(transparent)] -pub struct Argument(pub ResourceOrString); - -impl JsonSchema for Argument { - fn schema_name() -> std::string::String { - "Argument".to_owned() - } - - fn json_schema(gen: &mut SchemaGenerator) -> Schema { - let number = gen.subschema_for::(); - - let mut schema = ResourceOrString::json_schema(gen).into_object(); - schema.subschemas().any_of.as_mut().unwrap().push(number); - - schema.into() - } -} - -impl> From for Argument { - fn from(value: T) -> Self { - Argument(value.into()) - } -} - -impl Deref for Argument { - type Target = ResourceOrString; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -/// The element type and dimensions for a particular tensor. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -pub struct Type { - #[serde(rename = "type")] - pub name: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub dimensions: Vec, -} - -/// The name of a tensor. -/// -/// Typically something like "stage", or "stage.2" if the stage has multiple -/// outputs. -#[derive(Debug, Clone, PartialEq, Hash, Eq, Ord, PartialOrd)] -pub struct Input { - pub name: String, - pub index: Option, -} - -impl_json_schema_via_regex!( - Input, - INPUT_PATTERN, - r#" -The name of a tensor. - -Typically something like "stage", or "stage.2" if the stage has multiple outputs. -"# -); - -impl Input { - pub fn new( - name: impl Into, - index: impl Into>, - ) -> Self { - Input { - name: name.into(), - index: index.into(), - } - } -} - -static INPUT_PATTERN: Lazy = Lazy::new(|| { - Regex::new(r"^(?P[a-zA-Z_][\w-]*)(?:\.(?P\d+))?$").unwrap() -}); - -impl FromStr for Input { - type Err = Box; - - fn from_str(s: &str) -> Result { - let captures = INPUT_PATTERN - .captures(s) - .ok_or("Expected something like \"fft\" or \"fft.2\"")?; - - let name = &captures["name"]; - let index = captures.name("index").map(|m| { - m.as_str() - .parse::() - .expect("Guaranteed by the regex") - }); - - Ok(Input::new(name, index)) - } -} - -impl Display for Input { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self.index { - Some(index) => write!(f, "{}.{}", self.name, index), - None => write!(f, "{}", self.name), - } - } -} - -impl Serialize for Input { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.collect_str(self) - } -} - -impl<'de> Deserialize<'de> for Input { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let raw = Cow::::deserialize(deserializer)?; - Input::from_str(&raw).map_err(|e| D::Error::custom(e.to_string())) - } -} - -/// The declaration for a resource, typically something like a wordlist or -/// environment variable. -#[derive( - Debug, - Clone, - Default, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -#[serde(deny_unknown_fields)] -pub struct ResourceDeclaration { - /// A resource who's default value is specified inline. - pub inline: Option, - /// A resource who's default value is meant to be loaded from a file. - pub path: Option, - #[serde(rename = "type", default)] - pub ty: ResourceType, -} - -/// How the resource should be treated inside the Rune. -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -#[serde(rename_all = "kebab-case")] -pub enum ResourceType { - /// The resource should be treated like as a `&str`. - String, - /// The resource should be treated like a `&[u8]`. - Binary, -} - -impl Default for ResourceType { - fn default() -> Self { - ResourceType::String - } -} - -/// A reference to some [`ResourceDeclaration`]. It typically looks like -/// `$RESOURCE_NAME`. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ResourceName(pub String); - -impl_json_schema_via_regex!( - ResourceName, - RESOURCE_NAME_PATTERN, - r#" -A reference to some [`ResourceDeclaration`]. It typically looks like -`$RESOURCE_NAME`. -"# -); - -impl> From for ResourceName { - fn from(s: S) -> Self { - ResourceName(s.into()) - } -} - -impl FromStr for ResourceName { - type Err = Box; - - fn from_str(s: &str) -> Result { - if !s.starts_with('$') { - return Err("resource names always start with a \"$\"".into()); - } - - if !RESOURCE_NAME_PATTERN.is_match(s) { - return Err("should be a valid identifier".into()); - } - - Ok(ResourceName(s[1..].to_string())) - } -} - -impl Deref for ResourceName { - type Target = String; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Serialize for ResourceName { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - self.to_string().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for ResourceName { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let repr = Cow::::deserialize(deserializer)?; - - if !repr.starts_with('$') { - return Err(D::Error::custom( - "resource names always start with a \"$\"", - )); - } - - let name = &repr[1..]; - - if name.is_empty() { - Err(D::Error::custom("the resource name is empty")) - } else if !RESOURCE_NAME_PATTERN.is_match(name) { - Err(D::Error::custom("should be a valid identifier")) - } else { - Ok(ResourceName(name.to_string())) - } - } -} - -impl Display for ResourceName { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "${}", self.0) - } -} - -/// The image a Rune is based on. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, - schemars::JsonSchema, -)] -#[schemars(transparent)] -pub struct Image(String); - -impl Image { - pub fn runicos_base() -> Self { - Image(String::from("runicos/base")) - } -} - -impl FromStr for Image { - type Err = std::convert::Infallible; - - fn from_str(s: &str) -> Result { - Ok(Image(s.to_string())) - } -} - -impl Display for Image { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -#[cfg(test)] -mod tests { - use jsonschema::JSONSchema; - - use super::*; - - #[cfg(test)] - macro_rules! map { - // map-like - ($($k:ident : $v:expr),* $(,)?) => { - std::iter::Iterator::collect(IntoIterator::into_iter([ - $( - (String::from(stringify!($k)), $v) - ),* - ])) - }; - // set-like - ($($v:expr),* $(,)?) => { - std::iter::Iterator::collect(std::array::IntoIter::new([$($v,)*])) - }; - } - - #[cfg(test)] - macro_rules! ty { - ($type:ident [$($dim:expr),*]) => { - crate::parse::Type { - name: String::from(stringify!($type)), - dimensions: vec![ $($dim),*], - } - }; - ($type:ident) => { - crate::parse::Type { - name: String::from(stringify!($type)), - dimensions: vec![], - } - } - } - - #[test] - fn parse_normal_input_specifier() { - let src = "audio"; - let should_be = Input::new("audio", None); - - let got = Input::from_str(src).unwrap(); - - assert_eq!(got, should_be); - assert_eq!(got.to_string(), src); - } - - #[test] - fn input_specifier_with_tuple() { - let src = "audio.2"; - let should_be = Input::new("audio", 2); - - let got = Input::from_str(src).unwrap(); - - assert_eq!(got, should_be); - assert_eq!(got.to_string(), src); - } - - #[test] - fn parse_v1() { - let src = "version: 1\nimage: asdf\npipeline: {}"; - - let got = Document::parse(src).unwrap(); - - assert!(matches!(got, Document::V1 { .. })); - } - - #[test] - #[should_panic = "expected version to be 1"] - fn other_versions_are_an_error() { - let src = "image: asdf\nversion: 2\npipeline:"; - - let got = Document::parse(src).unwrap(); - - assert!(matches!(got, Document::V1 { .. })); - } - - #[test] - fn inline_resource() { - let src = "inline: some data"; - let should_be = ResourceDeclaration { - inline: Some(String::from("some data")), - ..Default::default() - }; - - let got: ResourceDeclaration = serde_yaml::from_str(src).unwrap(); - - assert_eq!(got, should_be); - } - - #[test] - fn resource_from_disk() { - let src = "path: ./input.txt"; - let should_be = ResourceDeclaration { - path: Some(String::from("./input.txt")), - ..Default::default() - }; - - let got: ResourceDeclaration = serde_yaml::from_str(src).unwrap(); - - assert_eq!(got, should_be); - } - - #[test] - fn resource_with_no_default_value() { - let src = "resource_name: {}"; - let should_be = ResourceDeclaration::default(); - - let got: IndexMap = - serde_yaml::from_str(src).unwrap(); - - let declaration = &got[0]; - assert_eq!(declaration, &should_be); - } - - #[test] - fn model_name_from_resource() { - let src = "$MODEL"; - let should_be = ResourceOrString::Resource("MODEL".into()); - - let got: ResourceOrString = serde_yaml::from_str(src).unwrap(); - - assert_eq!(got, should_be); - - let round_tripped = serde_yaml::to_string(&got).unwrap(); - assert_eq!(round_tripped, "---\n$MODEL\n"); - } - - #[test] - #[should_panic = "should be a valid identifier"] - fn model_name_from_resource_must_not_be_empty() { - let src = "$"; - - let _: ResourceOrString = serde_yaml::from_str(src).unwrap(); - } - - #[test] - #[should_panic = "should be a valid identifier"] - fn model_name_from_resource_must_be_valid_identifier() { - let src = "$"; - - let _: ResourceOrString = serde_yaml::from_str(src).unwrap(); - } - - #[test] - fn model_name_from_path() { - let src = "./path"; - let should_be = ResourceOrString::String(String::from(src)); - - let got: ResourceOrString = serde_yaml::from_str(src).unwrap(); - - assert_eq!(got, should_be); - - let round_tripped = serde_yaml::to_string(&got).unwrap(); - assert_eq!(round_tripped, "---\n\"./path\"\n"); - } - - #[test] - fn proc_block_with_resource_for_arg() { - let src = r#" - some-proc-block: - proc-block: normalize - outputs: - - type: u8 - dimensions: [1] - args: - word-list: $WORD_LIST - "#; - let should_be = Stage::ProcBlock(ProcBlockStage { - proc_block: "normalize".parse().unwrap(), - inputs: Vec::new(), - outputs: vec![Type { - name: String::from("u8"), - dimensions: vec![1], - }], - args: vec![( - "word-list".to_string(), - ResourceName::from_str("$WORD_LIST").unwrap().into(), - )] - .into_iter() - .collect(), - }); - - let got: IndexMap = serde_yaml::from_str(src).unwrap(); - - let got = &got["some-proc-block"]; - assert_eq!(got, &should_be); - } - - #[test] - fn parse_yaml_pipeline() { - let src = r#" -version: 1 -image: "runicos/base" - -pipeline: - audio: - capability: SOUND - outputs: - - type: i16 - dimensions: [16000] - args: - hz: 16000 - - fft: - proc-block: "git://github.com/hotg-ai/rune#proc_blocks/fft" - inputs: - - audio - outputs: - - type: i8 - dimensions: [1960] - - model: - model: "./model.tflite" - inputs: - - fft - outputs: - - type: i8 - dimensions: [6] - - label: - proc-block: "git://github.com/hotg-ai/rune#proc_blocks/ohv_label?tag=v0.11.3" - inputs: - - model - outputs: - - type: utf8 - args: - labels: | - silence - unknown - up - down - left - right - - output: - out: SERIAL - inputs: - - label - "#; - let should_be = Document::V1(DocumentV1 { - version: 1, - image: "runicos/base".parse().unwrap(), - pipeline: map! { - audio: Stage::Capability(CapabilityStage { - capability: String::from("SOUND"), - outputs: vec![ty!(i16[16000])], - args: map! { hz: "16000".into() }, - }), - fft: Stage::ProcBlock(ProcBlockStage { - proc_block: "git://github.com/hotg-ai/rune#proc_blocks/fft".parse().unwrap(), - inputs: vec!["audio".parse().unwrap()], - outputs: vec![ty!(i8[1960])], - args: IndexMap::new(), - }), - model: Stage::Model(ModelStage { - model: "./model.tflite".parse().unwrap(), - inputs: vec!["fft".parse().unwrap()], - outputs: vec![ty!(i8[6])], - args: IndexMap::new(), - }), - label: Stage::ProcBlock(ProcBlockStage { - proc_block: "git://github.com/hotg-ai/rune#proc_blocks/ohv_label?tag=v0.11.3".parse().unwrap(), - inputs: vec!["model".parse().unwrap()], - outputs: vec![Type { name: String::from("utf8"), dimensions: Vec::new() }], - args: map! { - labels: "silence\nunknown\nup\ndown\nleft\nright".into() - }, - }), - output: Stage::Out(OutStage { - out: String::from("SERIAL"), - args: IndexMap::new(), - inputs: vec!["label".parse().unwrap()], - }), - }, - resources: map![], - }); - - let got = Document::parse(src).unwrap(); - - assert_eq!(got, should_be); - } - - #[test] - fn parse_audio_block() { - let src = r#" - capability: SOUND - outputs: - - type: i16 - dimensions: [16000] - args: - hz: 16000 - "#; - let should_be = Stage::Capability(CapabilityStage { - capability: String::from("SOUND"), - outputs: vec![Type { - name: String::from("i16"), - dimensions: vec![16000], - }], - args: map! { hz: "16000".into() }, - }); - - let got: Stage = serde_yaml::from_str(src).unwrap(); - - assert_eq!(got, should_be); - } - - #[test] - fn schema_is_in_sync_with_version_on_disk() { - let filename = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("runefile-schema.json"); - let existing_schema = std::fs::read_to_string(&filename).unwrap(); - let existing_schema: serde_json::Value = - serde_json::from_str(&existing_schema).unwrap(); - - let schema = schemars::schema_for!(Document); - let current_schema = serde_json::to_value(&schema).unwrap(); - - if existing_schema != current_schema { - let serialized = - serde_json::to_string_pretty(¤t_schema).unwrap(); - std::fs::write(&filename, serialized.as_bytes()).unwrap(); - panic!("The runefile-schema.json was out of date"); - } - } - - #[track_caller] - fn handle_errors<'a>( - errors: impl Iterator>, - ) -> ! { - for err in errors { - println!("{}", err); - } - - panic!("Validation failed"); - } - - #[test] - fn argument_schema_is_valid() { - let schema = schemars::schema_for!(Argument); - let schema_json = serde_json::to_value(&schema).unwrap(); - let compiled_schema = - JSONSchema::options().compile(&schema_json).unwrap(); - - let string = serde_json::Value::String("".to_string()); - compiled_schema - .validate(&string) - .unwrap_or_else(|e| handle_errors(e)); - - let resource = serde_json::Value::String("$resource".to_string()); - compiled_schema - .validate(&resource) - .unwrap_or_else(|e| handle_errors(e)); - - let number = serde_json::Value::Number(10.into()); - compiled_schema - .validate(&number) - .unwrap_or_else(|e| handle_errors(e)); - } -} diff --git a/crates/compiler-2/src/type_check/mod.rs b/crates/compiler-2/src/type_check/mod.rs deleted file mode 100644 index 5fa6349073..0000000000 --- a/crates/compiler-2/src/type_check/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! The Rune compiler's type checker. - -mod types; - -pub use types::*; diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index a2660b0ef3..f9a5edf471 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -1,48 +1,29 @@ [package] name = "hotg-rune-compiler" version = "0.11.3" -authors = ["The Rune Developers "] -edition = "2018" -description = "Runefile parsing and analysis." +edition = "2021" +publish = false license = "MIT OR Apache-2.0" -homepage = "https://hotg.dev/" -repository = "https://github.com/hotg-ai/rune" -categories = ["science", "parser-implementations"] -keywords = ["rune", "tinyml", "container", "machine", "learning"] readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -atomic_refcell = "0.1.8" -bytes = { version = "1.1.0", features = ["serde"] } -cargo_toml = "0.10.3" -codespan = { version = "0.11.1", features = ["serialization"] } -codespan-reporting = "0.11.1" -heck = "0.4.0" -hotg-rune-core = { path = "../rune-core", version = "^0.11.0"} -hotg-rune-proc-blocks = { path = "../proc-blocks", version = "^0.11.0", default-features = false } -im = { version = "15.0.0", features = ["serde"] } -indexmap = { version = "1.8.0", features = ["serde-1"] } -indoc = "1.0.3" -legion = { version = "0.4.0", default-features = false, features = ["serialize", "codegen", "extended-tuple-impls"] } -log = "0.4.14" -miette = "4.4.0" -once_cell = "1.9.0" -proc-macro2 = "1.0.36" -quote = "1.0.14" -regex = "1.5.4" +indexmap = { version = "1.8.1", features = ["serde-1"] } +once_cell = "1.10.0" +regex = "1.5.5" salsa = "0.16.1" schemars = { version = "0.8.8", features = ["indexmap"] } -serde = { version = "1.0.133", features = ["derive"] } -serde_json = "1.0.74" +serde = "1.0.136" +serde_json = "1.0.79" serde_yaml = "0.8.23" thiserror = "1.0.30" -toml = "0.5.8" -tracing = { version = "0.1.34", features = ["attributes"] } -zip = "0.5.13" +tracing = "0.1.34" +uriparse = "0.6.4" +zip = "0.6.2" [dev-dependencies] -env_logger = "0.9.0" -jsonschema = { version = "0.16.0", default-features = false } -pretty_assertions = "1.0.0" +insta = "1.14.0" +jsonschema = "0.15.2" +pretty_assertions = "1.2.1" +tracing-test = "0.2.1" diff --git a/crates/compiler/error-codes.md b/crates/compiler/error-codes.md deleted file mode 100644 index e116a71a45..0000000000 --- a/crates/compiler/error-codes.md +++ /dev/null @@ -1,9 +0,0 @@ -# Error Codes - - -## E001: Parse Failed - - - -## E002: A resource can't specify both "path" and "inline" values - diff --git a/crates/compiler/examples/extensions.rs b/crates/compiler/examples/extensions.rs deleted file mode 100644 index 357755cbd1..0000000000 --- a/crates/compiler/examples/extensions.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! An example showing how you might hook into the Rune build process. -//! -//! This example achieves a couple things: -//! -//! 1. After the YAML document is analysed we emit a warning for every [`Model`] -//! that it includes -//! 2. We implement dotenv-like functionality by checking if any resources (e.g. -//! `foo`) have the corresponding environment variable set (e.g. `$FOO`). If -//! so, the resource's [`ResourceData`] component is overridden with its -//! value. -//! 3. Print out some any diagnostics at the end so we can see the effects from -//! step 1. - -use std::{fmt::Write as _, path::Path}; - -use codespan_reporting::diagnostic::{Diagnostic, Severity}; -use env_logger::Env; -use hotg_rune_compiler::{ - hooks::{ - AfterCodegenContext, AfterLoweringContext, AfterTypeCheckingContext, - Continuation, Hooks, - }, - lowering::{Model, Name, Resource, ResourceData}, - BuildContext, Diagnostics, FeatureFlags, -}; -use legion::{component, systems::CommandBuffer, Entity, IntoQuery}; - -fn main() { - env_logger::init_from_env(Env::new().default_filter_or("debug")); - - let directory = std::env::args().nth(1).expect("Usage: ./extensions "); - let mut build_ctx = BuildContext::for_directory(directory) - .expect("Couldn't read the Runefile"); - - build_ctx.working_directory = - project_root().join("target").join("extensions-working-dir"); - - let mut hooks = CustomHooks::default(); - - let (_world, res) = hotg_rune_compiler::build_with_hooks( - build_ctx, - FeatureFlags::development(), - &mut hooks, - ); - - // Print out all diagnostics. Normally you'd use the codespan_reporting - // crate, but println!() is good enough for now. - let diags = res.get::().unwrap(); - - log::info!("Printing {} diagnostics...", diags.len()); - for diag in diags.iter() { - let level = match diag.severity { - Severity::Bug | Severity::Error => log::Level::Error, - Severity::Warning => log::Level::Warn, - _ => log::Level::Info, - }; - log::log!(level, "{:?}: {}", diag.severity, diag.message); - } -} - -#[derive(Debug, Default)] -struct CustomHooks {} - -impl Hooks for CustomHooks { - fn after_lowering( - &mut self, - ctx: &mut dyn AfterLoweringContext, - ) -> Continuation { - warn_about_every_model_in_the_rune(ctx); - Continuation::Continue - } - - fn after_type_checking( - &mut self, - ctx: &mut dyn AfterTypeCheckingContext, - ) -> Continuation { - dotenv(ctx); - Continuation::Continue - } - - fn after_codegen( - &mut self, - ctx: &mut dyn AfterCodegenContext, - ) -> Continuation { - for file in - <&hotg_rune_compiler::codegen::File>::query().iter(ctx.world()) - { - let mut msg = String::new(); - - match core::str::from_utf8(&file.data) { - Ok(string) => { - for line in string.lines() { - writeln!(msg, "\t{}", line).unwrap(); - } - }, - Err(_) => writeln!(msg, "\t(binary)").unwrap(), - } - log::info!("Reading: {}\n{}", file.path.display(), msg); - } - - Continuation::Continue - } -} - -fn warn_about_every_model_in_the_rune(ctx: &mut dyn AfterLoweringContext) { - let mut diags = ctx.diagnostics_mut(); - let mut model_names = <&Name>::query().filter(component::()); - - for name in model_names.iter(ctx.world()) { - let msg = format!("The Rune contains a model called \"{}\"", name); - diags.push(Diagnostic::warning().with_message(msg)); - } -} - -/// Implement `dotenv`-like behaviour by looking for the environment variable -/// that corresponds to a particular [`Resource`] and setting its -/// [`ResourceData`] if that variable is set. -fn dotenv(ctx: &mut dyn AfterTypeCheckingContext) { - let (world, res) = ctx.world_and_resources(); - - let mut cmd = CommandBuffer::new(world); - - // create a query which will look for all named entities with a "Resource" - // component. - let mut query = <(Entity, &Name)>::query().filter(component::()); - - for (&ent, name) in query.iter(world) { - let variable_name = name.to_uppercase(); - - if let Ok(value) = std::env::var(variable_name) { - println!( - "Overriding the \"{}\" resource and setting it to \"{}\"", - name, value - ); - cmd.add_component(ent, ResourceData::from(value.into_bytes())); - } - } - - cmd.flush(world, res); -} - -fn project_root() -> &'static Path { - for ancestor in Path::new(env!("CARGO_MANIFEST_DIR")).ancestors() { - if ancestor.join(".git").exists() { - return ancestor; - } - } - - panic!("Unable to determine the project's root directory"); -} diff --git a/crates/compiler/runefile-schema.json b/crates/compiler/runefile-schema.json index 7afb800d88..522d39b6b9 100644 --- a/crates/compiler/runefile-schema.json +++ b/crates/compiler/runefile-schema.json @@ -1,7 +1,5 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Document", - "description": "The top level Runefile type.", "anyOf": [ { "$ref": "#/definitions/DocumentV1" @@ -9,7 +7,6 @@ ], "definitions": { "Argument": { - "description": "Something that could be either a reference to a resource (`$resource`) or a plain string (`./path`).", "anyOf": [ { "$ref": "#/definitions/ResourceName" @@ -20,187 +17,169 @@ { "type": "number" } - ] + ], + "description": "Something that could be either a reference to a resource (`$resource`) or a plain string (`./path`)." }, "CapabilityStage": { "description": "A stage which reads inputs from the runtime.", - "type": "object", - "required": [ - "capability" - ], "properties": { "args": { - "type": "object", "additionalProperties": { "$ref": "#/definitions/Argument" - } + }, + "type": "object" }, "capability": { "description": "What type of capability to use (\"IMAGE\", \"SOUND\", etc.).", "type": "string" }, "outputs": { - "type": "array", "items": { "$ref": "#/definitions/Type" - } + }, + "type": "array" } - } + }, + "required": [ + "capability" + ], + "type": "object" }, "DocumentV1": { "description": "Version 1 of the `Runefile.yml` format.", - "type": "object", - "required": [ - "image", - "pipeline", - "version" - ], "properties": { "image": { "description": "The base image that defines the interface between a Rune and its runtime.\n\nThis should always be `\"runicos/base\"`.", - "allOf": [ - { - "$ref": "#/definitions/Path" - } - ] + "type": "string" }, "pipeline": { - "description": "The various stages in the Runefile's pipeline.", - "type": "object", "additionalProperties": { "$ref": "#/definitions/Stage" - } + }, + "description": "The various stages in the Runefile's pipeline.", + "type": "object" }, "resources": { - "description": "Any resources that can be accessed by pipeline stages.", - "default": {}, - "type": "object", "additionalProperties": { "$ref": "#/definitions/ResourceDeclaration" - } + }, + "default": {}, + "description": "Any resources that can be accessed by pipeline stages.", + "type": "object" }, "version": { "description": "The version number. Must always be `\"1\"`.", - "type": "integer", "format": "uint", "maximum": 1.0, - "minimum": 1.0 + "minimum": 1.0, + "type": "integer" } - } + }, + "required": [ + "image", + "pipeline", + "version" + ], + "type": "object" }, "Input": { "description": "\nThe name of a tensor.\n\nTypically something like \"stage\", or \"stage.2\" if the stage has multiple outputs.\n", - "type": "string", "format": "string", - "pattern": "^(?P[a-zA-Z_][\\w-]*)(?:\\.(?P\\d+))?$" + "pattern": "^(?P[a-zA-Z_][\\w-]*)(?:\\.(?P\\d+))?$", + "type": "string" }, "ModelStage": { "description": "A ML model which will be executed by the runtime.", - "type": "object", - "required": [ - "model" - ], "properties": { "args": { - "type": "object", "additionalProperties": { "$ref": "#/definitions/Argument" - } + }, + "type": "object" }, "inputs": { "description": "Tensors to use as input to this model.", - "type": "array", "items": { "$ref": "#/definitions/Input" - } + }, + "type": "array" }, "model": { "description": "The model to use, or a resource which specifies the model to use.", - "anyOf": [ - { - "$ref": "#/definitions/ResourceName" - }, - { - "type": "string" - } - ] + "type": "string" }, "outputs": { "description": "The tensors that this model outputs.", - "type": "array", "items": { "$ref": "#/definitions/Type" - } + }, + "type": "array" } - } + }, + "required": [ + "model" + ], + "type": "object" }, "OutStage": { "description": "A stage which passes outputs back to the runtime.", - "type": "object", - "required": [ - "out" - ], "properties": { "args": { - "type": "object", "additionalProperties": { "$ref": "#/definitions/Argument" - } + }, + "type": "object" }, "inputs": { - "type": "array", "items": { "$ref": "#/definitions/Input" - } + }, + "type": "array" }, "out": { "description": "The type of output (e.g. \"SERIAL\").", "type": "string" } - } - }, - "Path": { - "description": "\nA specification for finding a dependency.\n\nThe full syntax is `base@version#sub_path` where\n\n- `base` is a URL or the name of a repository on GitHub (e.g. `hotg-ai/rune`\n or `https://github.com/hotg-ai/rune`)\n- `version` is an optional field specifying the version (e.g. as a git tag)\n- `sub_path` is an optional field which is useful when pointing to\n repositories with multiple relevant items because it lets you specify\n which directory the specified item is in.\n", - "type": "string", - "format": "string", - "pattern": "(?x)\n (?P[\\w\\d:/_.-]+)\n (?:@(?P[\\w\\d./-]+))?\n (?:\\#(?P[\\w\\d._/-]+))?\n " + }, + "required": [ + "out" + ], + "type": "object" }, "ProcBlockStage": { "description": "A stage which executes a procedural block.", - "type": "object", - "required": [ - "proc-block" - ], "properties": { "args": { - "type": "object", "additionalProperties": { "$ref": "#/definitions/Argument" - } + }, + "type": "object" }, "inputs": { - "type": "array", "items": { "$ref": "#/definitions/Input" - } + }, + "type": "array" }, "outputs": { - "type": "array", "items": { "$ref": "#/definitions/Type" - } + }, + "type": "array" }, "proc-block": { "description": "A [`Path`] that Rune can use to locate the proc block.", - "type": "string", - "format": "string", - "pattern": "(?x)\n (?P[\\w\\d:/_.-]+)\n (?:@(?P[\\w\\d./-]+))?\n (?:\\#(?P[\\w\\d._/-]+))?\n " + "type": "string" } - } + }, + "required": [ + "proc-block" + ], + "type": "object" }, "ResourceDeclaration": { + "additionalProperties": false, "description": "The declaration for a resource, typically something like a wordlist or environment variable.", - "type": "object", "properties": { "inline": { "description": "A resource who's default value is specified inline.", @@ -217,32 +196,31 @@ ] }, "type": { - "default": "string", "allOf": [ { "$ref": "#/definitions/ResourceType" } - ] + ], + "default": "string" } }, - "additionalProperties": false + "type": "object" }, "ResourceName": { "description": "\nA reference to some [`ResourceDeclaration`]. It typically looks like\n`$RESOURCE_NAME`.\n", - "type": "string", "format": "string", - "pattern": "^\\$[_a-zA-Z][_a-zA-Z0-9]*$" + "pattern": "^\\$[_a-zA-Z][_a-zA-Z0-9]*$", + "type": "string" }, "ResourceType": { "description": "How the resource should be treated inside the Rune.", - "type": "string", "enum": [ "string", "binary" - ] + ], + "type": "string" }, "Stage": { - "description": "A stage in the Rune's pipeline.", "anyOf": [ { "$ref": "#/definitions/ModelStage" @@ -256,27 +234,30 @@ { "$ref": "#/definitions/OutStage" } - ] + ], + "description": "A stage in the Rune's pipeline." }, "Type": { "description": "The element type and dimensions for a particular tensor.", - "type": "object", - "required": [ - "type" - ], "properties": { "dimensions": { - "type": "array", "items": { - "type": "integer", "format": "uint", - "minimum": 0.0 - } + "minimum": 0.0, + "type": "integer" + }, + "type": "array" }, "type": { "type": "string" } - } + }, + "required": [ + "type" + ], + "type": "object" } - } -} + }, + "description": "The top level Runefile type.", + "title": "Document" +} \ No newline at end of file diff --git a/crates/compiler/src/build_context.rs b/crates/compiler/src/build_context.rs deleted file mode 100644 index 81a2da3928..0000000000 --- a/crates/compiler/src/build_context.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::{ - path::{Path, PathBuf}, - process::Command, -}; - -use crate::codegen::RuneVersion; - -/// Inputs used during the compilation process. -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct BuildContext { - /// The name of the Rune being compiled. - pub name: String, - /// The `Runefile.yml` source text. - pub runefile: String, - /// A directory that can be used for any temporary artifacts. - pub working_directory: PathBuf, - /// The directory that all paths (e.g. to models) are resolved relative to. - pub current_directory: PathBuf, - /// Generate an optimized build. - pub optimized: bool, - pub verbosity: Verbosity, - /// The version of Rune being used. - pub rune_version: Option, -} - -impl BuildContext { - /// Create a new [`BuildContext`] using the convention that the - /// [`BuildContext.name`] is named after the - /// [`BuildContext.current_directory`]. - pub fn for_directory( - directory: impl Into, - ) -> Result { - let current_directory = directory.into(); - let working_directory = current_directory.clone(); - - let name = current_directory - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::Other, - "Unable to determine the current directory's name", - ) - })?; - - let runefile = current_directory.join("Runefile.yml"); - let runefile = std::fs::read_to_string(runefile)?; - - Ok(BuildContext { - name, - runefile, - working_directory, - current_directory, - optimized: true, - verbosity: Verbosity::Normal, - rune_version: Some(RuneVersion { - version: env!("CARGO_PKG_VERSION").to_string(), - }), - }) - } - - #[cfg(test)] - pub(crate) fn from_doc(doc: crate::parse::Document) -> Self { - BuildContext { - name: "rune".to_string(), - runefile: serde_yaml::to_string(&doc).unwrap(), - working_directory: PathBuf::from("."), - current_directory: PathBuf::from("."), - optimized: false, - verbosity: Verbosity::Normal, - rune_version: Some(RuneVersion { - version: env!("CARGO_PKG_VERSION").to_string(), - }), - } - } -} - -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub enum Verbosity { - Quiet, - Normal, - Verbose, -} - -impl Verbosity { - pub fn from_quiet_and_verbose(quiet: bool, verbose: bool) -> Option { - match (verbose, quiet) { - (true, false) => Some(Verbosity::Verbose), - (false, true) => Some(Verbosity::Quiet), - (false, false) => Some(Verbosity::Normal), - (true, true) => None, - } - } - - /// Add a `--quiet` or `--verbose` argument to the command if necessary. - pub fn add_flags(&self, cmd: &mut Command) { - match self { - Verbosity::Quiet => { - cmd.arg("--quiet"); - }, - Verbosity::Verbose => { - cmd.arg("--verbose"); - }, - Verbosity::Normal => {}, - } - } -} - -/// Feature flags and other knobs that can be used during development. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FeatureFlags { - pub(crate) rune_repo_dir: Option, -} - -impl FeatureFlags { - pub fn development() -> Self { - let hotg_repo_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .and_then(Path::parent) - .filter(|repo_root| repo_root.join(".git").exists()) - .map(PathBuf::from); - - FeatureFlags { - rune_repo_dir: hotg_repo_dir, - } - } - - pub const fn production() -> Self { - FeatureFlags { - rune_repo_dir: None, - } - } - - /// If specified, Rune crates (e.g `hotg-rune-core`) will be patched - /// to use crates from this directory instead of crates.io or GitHub. - pub fn set_rune_repo_dir( - &mut self, - hotg_repo_dir: impl Into>, - ) -> &mut Self { - self.rune_repo_dir = hotg_repo_dir.into(); - self - } -} - -impl Default for FeatureFlags { - fn default() -> Self { FeatureFlags::production() } -} diff --git a/crates/compiler/src/codegen/compile_generated_project.rs b/crates/compiler/src/codegen/compile_generated_project.rs deleted file mode 100644 index 64e0c54c48..0000000000 --- a/crates/compiler/src/codegen/compile_generated_project.rs +++ /dev/null @@ -1,2 +0,0 @@ -#[legion::system] -pub(crate) fn run() {} diff --git a/crates/compiler/src/codegen/components.rs b/crates/compiler/src/codegen/components.rs deleted file mode 100644 index 4b3065faa0..0000000000 --- a/crates/compiler/src/codegen/components.rs +++ /dev/null @@ -1,193 +0,0 @@ -use std::{ - collections::HashMap, - fmt::{self, Display, Formatter}, - ops::Deref, - path::PathBuf, -}; - -use bytes::Bytes; -use hotg_rune_core::Shape; -use serde::Serialize; - -use crate::{ - lowering::{Name, Resource, SinkKind, SourceKind}, - parse::{Path, ResourceOrString}, -}; - -pub const GRAPH_CUSTOM_SECTION: &str = ".rune_graph"; -pub const VERSION_CUSTOM_SECTION: &str = ".rune_version"; -pub const RESOURCE_CUSTOM_SECTION: &str = ".rune_resource"; - -/// A file that will be written to the Rune's build directory. -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct File { - pub path: PathBuf, - pub data: Bytes, -} - -impl File { - pub fn new(path: impl Into, data: impl Into) -> Self { - File { - path: path.into(), - data: data.into(), - } - } -} - -/// A WebAssembly custom section to be embedded in the Rune. -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct CustomSection { - pub section_name: String, - pub value: Bytes, -} - -impl CustomSection { - pub fn new(name: impl Into, value: impl Into) -> Self { - let section_name = name.into(); - let value = value.into(); - - debug_assert!( - section_name.starts_with('.'), - "Link section names should start with a \".\", found \"{}\"", - section_name - ); - - CustomSection { - section_name, - value, - } - } - - pub fn from_json( - name: impl Into, - value: &impl Serialize, - ) -> Result { - let value = serde_json::to_vec(value)?; - let name = name.into(); - Ok(CustomSection::new(name, value)) - } - - pub(crate) fn identifier(&self) -> &str { - self.section_name.trim_start_matches('.') - } -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct RuneVersion { - /// The version of the tool generating a Rune, typically what you'd see - /// when running `rune --version`. - pub version: String, -} - -impl RuneVersion { - pub fn new(version: impl Into) -> Self { - RuneVersion { - version: version.into(), - } - } - - pub(crate) fn as_custom_section( - &self, - ) -> Result { - CustomSection::from_json(VERSION_CUSTOM_SECTION, self) - } -} - -impl Display for RuneVersion { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.write_str("v")?; - f.write_str(&self.version)?; - Ok(()) - } -} - -/// A summary of the Rune pipeline that will be embedded in the Rune. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct RuneGraph { - pub rune: RuneSummary, - #[serde(skip_serializing_if = "HashMap::is_empty", default)] - pub capabilities: HashMap, - #[serde(skip_serializing_if = "HashMap::is_empty", default)] - pub models: HashMap, - #[serde(skip_serializing_if = "HashMap::is_empty", default)] - pub proc_blocks: HashMap, - #[serde(skip_serializing_if = "HashMap::is_empty", default)] - pub outputs: HashMap, - #[serde(skip_serializing_if = "HashMap::is_empty", default)] - pub resources: HashMap, - #[serde(skip_serializing_if = "HashMap::is_empty", default)] - pub tensors: HashMap>, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct RuneSummary { - pub name: String, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct CapabilitySummary { - pub kind: SourceKind, - pub args: HashMap, - pub outputs: Vec, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct ModelSummary { - pub file: ResourceOrString, - pub args: HashMap, - pub inputs: Vec, - pub outputs: Vec, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct ProcBlockSummary { - pub path: Path, - pub args: HashMap, - pub inputs: Vec, - pub outputs: Vec, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct OutputSummary { - pub kind: SinkKind, - pub args: HashMap, - pub inputs: Vec, -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub struct TensorId(pub String); - -impl From for TensorId { - fn from(s: String) -> Self { TensorId(s) } -} - -impl Deref for TensorId { - type Target = String; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -impl RuneGraph { - pub(crate) fn as_custom_section( - &self, - ) -> Result { - CustomSection::from_json(GRAPH_CUSTOM_SECTION, self) - } -} diff --git a/crates/compiler/src/codegen/generate_cargo_config.rs b/crates/compiler/src/codegen/generate_cargo_config.rs deleted file mode 100644 index c5f39160fa..0000000000 --- a/crates/compiler/src/codegen/generate_cargo_config.rs +++ /dev/null @@ -1,110 +0,0 @@ -use legion::systems::CommandBuffer; - -use crate::{codegen::File, BuildContext}; - -/// Generate a `.cargo/config.toml` file. -#[legion::system] -pub(crate) fn run(cmd: &mut CommandBuffer, #[resource] ctx: &BuildContext) { - let config = generate_config(ctx.optimized); - cmd.push((config,)); -} - -pub(crate) fn generate_config(optimized: bool) -> File { - let target = if optimized { - Some(Targets { - wasm32_unknown_unknown: Target { - rustflags: &["-C", "link-arg=-s"], - }, - }) - } else { - None - }; - - let config = Config { - target, - net: Net { - git_fetch_with_cli: true, - }, - build: Build { - target: "wasm32-unknown-unknown", - }, - }; - - let config = toml::to_vec(&config) - .expect("We can always serialize a Config to TOML"); - - File::new(".cargo/config.toml", config) -} - -#[derive(Debug, serde::Serialize)] -struct Config { - target: Option, - net: Net, - build: Build, -} - -/// The [`[build]`](https://doc.rust-lang.org/cargo/reference/config.html#build) -/// table. -#[derive(Debug, serde::Serialize)] -struct Build { - /// The default target triple. - target: &'static str, -} - -/// The `[target]` table. -#[derive(Debug, serde::Serialize)] -#[serde(rename_all = "kebab-case")] -struct Targets { - wasm32_unknown_unknown: Target, -} - -#[derive(Debug, serde::Serialize)] -struct Target { - rustflags: &'static [&'static str], -} - -#[derive(Debug, serde::Serialize)] -#[serde(rename_all = "kebab-case")] -struct Net { - git_fetch_with_cli: bool, -} - -#[cfg(test)] -mod tests { - use toml::Value; - - use super::*; - - #[test] - fn request_small_binaries_when_optimised() { - let should_be = toml::toml! { - [target.wasm32-unknown-unknown] - rustflags = ["-C", "link-arg=-s"] - - [net] - git-fetch-with-cli = true - - [build] - target = "wasm32-unknown-unknown" - }; - - let got = generate_config(true); - - assert_eq!(toml::from_slice::(&got.data).unwrap(), should_be); - } - - #[test] - fn only_git_fetch_with_cli_for_debug_builds() { - let should_be = toml::toml! { - [net] - git-fetch-with-cli = true - - [build] - target = "wasm32-unknown-unknown" - }; - - let got = generate_config(false); - - assert_eq!(toml::from_slice::(&got.data).unwrap(), should_be); - } -} diff --git a/crates/compiler/src/codegen/generate_cargo_toml.rs b/crates/compiler/src/codegen/generate_cargo_toml.rs deleted file mode 100644 index fb764135c4..0000000000 --- a/crates/compiler/src/codegen/generate_cargo_toml.rs +++ /dev/null @@ -1,387 +0,0 @@ -use std::{collections::BTreeMap, path::Path}; - -use cargo_toml::{ - Badges, Dependency, DependencyDetail, DepsSet, Edition, FeatureSet, - Manifest, Package, PatchSet, Product, Profiles, Publish, Resolver, - TargetDepsSet, Workspace, -}; -use legion::{systems::CommandBuffer, world::SubWorld, Query}; - -use crate::{ - codegen::File, lowering::ProcBlock, parse, BuildContext, FeatureFlags, -}; - -/// Generate a `Cargo.toml` file which includes all the relevant dependencies -/// for this crate. -#[legion::system] -pub(crate) fn run( - world: &SubWorld, - cmd: &mut CommandBuffer, - #[resource] ctx: &BuildContext, - #[resource] features: &FeatureFlags, - query: &mut Query<&ProcBlock>, -) { - let proc_blocks: Vec<_> = query.iter(world).collect(); - - let file = generate(features, proc_blocks.iter().map(|&p| p), ctx); - cmd.push((file,)); -} - -pub(crate) fn generate<'a>( - features: &FeatureFlags, - proc_blocks: impl Iterator + 'a, - ctx: &BuildContext, -) -> File { - let core_version = hotg_rune_core::VERSION; - - if core_version.contains("-dev") && features.rune_repo_dir.is_none() { - let msg = indoc::indoc!( - " - It looks like you are using a development version of \"rune\", but - haven't specified a \"rune_repo_dir\". Internal crates are resolved - using the \"$CORE_VERSION\" version from crates.io and builtin - proc-blocks are found using the \"v$CORE_VERSION\" tag from the \ - Rune - repo, so there is a good chance you'll get compile errors about - unresolved dependencies. Specify the \"rune_repo_dir\" to resolve - this. - " - ); - log::warn!( - "{}", - msg.replace("\n", " ") - .replace("$CORE_VERSION", core_version) - ); - } - - let mut manifest = - generate_manifest(proc_blocks, &ctx.name, &ctx.current_directory); - - if let Some(hotg_repo_dir) = features.rune_repo_dir.as_deref() { - patch_hotg_dependencies(hotg_repo_dir, &mut manifest); - } - - let manifest = toml::to_string_pretty(&manifest) - .expect("Serializing to a string should never fail"); - - File::new("Cargo.toml", manifest.into_bytes()) -} - -// Generate the `Cargo.toml` manifest. -fn generate_manifest<'rune, I>( - proc_blocks: I, - name: &str, - current_dir: &Path, -) -> Manifest -where - I: IntoIterator + 'rune, -{ - let product = Product { - path: Some("lib.rs".to_string()), - edition: Some(Edition::E2018), - crate_type: Some(vec!["cdylib".to_string()]), - ..Default::default() - }; - - Manifest { - package: Some(package(name)), - lib: Some(product), - dependencies: dependencies(proc_blocks, current_dir), - workspace: Some(Workspace { - members: vec![String::from(".")], - default_members: vec![String::from(".")], - exclude: Vec::new(), - metadata: None, - }), - ..empty_manifest() - } -} - -fn package(name: &str) -> Package { - Package { - name: name.into(), - edition: Edition::E2018, - version: String::from("0.0.0"), - publish: Publish::Flag(false), - resolver: Some(Resolver::V2), - ..empty_package() - } -} - -fn dependencies<'rune, I>(proc_blocks: I, current_dir: &Path) -> DepsSet -where - I: IntoIterator + 'rune, -{ - let mut deps = DepsSet::new(); - - // We always need the log crate - let log = Dependency::Detailed(DependencyDetail { - version: Some(String::from("0.4")), - features: vec![ - String::from("max_level_debug"), - String::from("release_max_level_debug"), - ], - ..empty_dependency_detail() - }); - deps.insert(String::from("log"), log); - - // we also need lazy_static - let lazy_static = Dependency::Detailed(DependencyDetail { - version: Some(String::from("1.0")), - features: vec![String::from("spin_no_std")], - ..empty_dependency_detail() - }); - deps.insert(String::from("lazy_static"), lazy_static); - - // We'll always use the following HOTG dependencies. - deps.insert( - "hotg-rune-core".to_string(), - Dependency::Simple(format!("^{}", hotg_rune_core::VERSION)), - ); - deps.insert( - "hotg-rune-proc-blocks".to_string(), - Dependency::Simple(format!("^{}", hotg_rune_proc_blocks::VERSION)), - ); - // FIXME: We should probably use the actual version number instead of - // assuming it'll be in sync with core. - deps.insert( - "hotg-runicos-base-wasm".to_string(), - Dependency::Simple(format!("^{}", hotg_rune_core::VERSION)), - ); - - for proc_block in proc_blocks { - let dep = proc_block_dependency(&proc_block.path, current_dir); - let name = proc_block.name(); - deps.insert(name.to_string(), Dependency::Detailed(dep)); - } - - deps -} - -fn proc_block_dependency( - path: &parse::Path, - current_dir: &Path, -) -> DependencyDetail { - if path.base.starts_with('.') { - return local_proc_block(path, current_dir); - } - - if path.sub_path.is_none() && !path.base.contains('/') { - if let Some(version) = &path.version { - // it's from crates.io - return DependencyDetail { - version: Some(version.clone()), - ..empty_dependency_detail() - }; - } - } - - // fall back to using git - let repo = format!("https://github.com/{}.git", path.base); - - DependencyDetail { - git: Some(repo), - rev: path.version.clone(), - ..empty_dependency_detail() - } -} - -fn local_proc_block( - path: &parse::Path, - current_dir: &Path, -) -> DependencyDetail { - DependencyDetail { - path: Some(current_dir.join(&path.base).display().to_string()), - ..empty_dependency_detail() - } -} - -fn empty_manifest() -> Manifest { - Manifest { - package: None, - dependencies: DepsSet::default(), - lib: None, - workspace: None, - dev_dependencies: DepsSet::default(), - build_dependencies: DepsSet::default(), - target: TargetDepsSet::default(), - features: FeatureSet::default(), - patch: PatchSet::default(), - profile: Profiles::default(), - badges: Badges::default(), - bin: Vec::default(), - bench: Vec::default(), - test: Vec::default(), - example: Vec::default(), - } -} - -fn empty_package() -> Package { - Package { - name: String::default(), - edition: Edition::default(), - version: String::default(), - build: None, - workspace: None, - authors: Default::default(), - links: None, - description: None, - homepage: None, - documentation: None, - readme: None, - keywords: Vec::new(), - categories: Vec::new(), - license: None, - license_file: None, - repository: None, - metadata: None, - default_run: None, - autobins: false, - autoexamples: false, - autotests: false, - autobenches: false, - publish: Publish::default(), - resolver: None, - } -} - -fn empty_dependency_detail() -> DependencyDetail { - DependencyDetail { - version: None, - registry: None, - registry_index: None, - path: None, - git: None, - branch: None, - tag: None, - rev: None, - features: Vec::new(), - optional: false, - default_features: None, - package: None, - } -} - -fn path_dependency(path: impl AsRef) -> Dependency { - Dependency::Detailed(DependencyDetail { - path: Some(path.as_ref().to_string_lossy().into()), - ..empty_dependency_detail() - }) -} - -fn patch_hotg_dependencies(hotg_repo_dir: &Path, manifest: &mut Manifest) { - let mut overrides = BTreeMap::new(); - - overrides.insert( - "hotg-rune-core".to_string(), - path_dependency(hotg_repo_dir.join("crates").join("rune-core")), - ); - overrides.insert( - "hotg-rune-proc-blocks".to_string(), - path_dependency(hotg_repo_dir.join("crates").join("proc-blocks")), - ); - overrides.insert( - "hotg-runicos-base-wasm".to_string(), - path_dependency( - hotg_repo_dir - .join("images") - .join("runicos-base") - .join("wasm"), - ), - ); - - // Patch crates.io - manifest - .patch - .entry("crates-io".to_string()) - .or_default() - .extend(overrides.clone()); - // Sometimes we'll pull from GitHub, so patch that too - manifest - .patch - .entry("https://github.com/hotg-ai/rune".to_string()) - .or_default() - .extend(overrides.clone()); - - // What can sometimes happen is that we'll use hotg_rune_core::VERSION as - // the version requirement, but if we are patching the hotg-XXX crate - // versions there's a good chance hotg_rune_core::VERSION doesn't exist on - // crates.io yet. - // - // If so, just override with the patched version. - manifest.dependencies.extend(overrides); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn base_dependencies() { - let got = dependencies(Vec::new(), Path::new(".")); - - assert_eq!(got.len(), 5); - assert!(got.contains_key("log")); - assert!(got.contains_key("lazy_static")); - assert!(got.contains_key("hotg-rune-core")); - assert!(got.contains_key("hotg-rune-proc-blocks")); - assert!(got.contains_key("hotg-runicos-base-wasm")); - - assert_eq!( - got["hotg-rune-core"].clone(), - Dependency::Simple(format!("^{}", hotg_rune_core::VERSION)) - ); - assert_eq!( - got["hotg-rune-proc-blocks"].clone(), - Dependency::Simple(format!("^{}", hotg_rune_proc_blocks::VERSION)) - ); - assert_eq!( - got["hotg-runicos-base-wasm"].clone(), - Dependency::Simple(format!("^{}", hotg_rune_core::VERSION)) - ); - } - - #[test] - fn proc_block_from_crates_io() { - let path = "whatever@1.2".parse().unwrap(); - let should_be = DependencyDetail { - version: Some("1.2".to_string()), - ..empty_dependency_detail() - }; - - let got = proc_block_dependency(&path, Path::new(".")); - - assert_eq!(got, should_be); - } - - #[test] - fn git_proc_block_with_version() { - let path = "organisation/whatever@1.2".parse().unwrap(); - let should_be = DependencyDetail { - git: Some( - "https://github.com/organisation/whatever.git".to_string(), - ), - rev: Some("1.2".to_string()), - ..empty_dependency_detail() - }; - - let got = proc_block_dependency(&path, Path::new(".")); - - assert_eq!(got, should_be); - } - - #[test] - fn manifest_generates_cdylib() { - let got = generate_manifest(Vec::new(), "foo", Path::new(".")); - - let crate_type = got.lib.unwrap().crate_type.unwrap(); - assert!(crate_type.contains(&String::from("cdylib"))); - } - - #[test] - fn manifest_is_in_its_own_workspace() { - let got = generate_manifest(Vec::new(), "foo", Path::new(".")); - - assert!(got.workspace.is_some()); - } -} diff --git a/crates/compiler/src/codegen/generate_lib_rs.rs b/crates/compiler/src/codegen/generate_lib_rs.rs deleted file mode 100644 index 0925164e57..0000000000 --- a/crates/compiler/src/codegen/generate_lib_rs.rs +++ /dev/null @@ -1,1215 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use heck::{ToSnakeCase, ToUpperCamelCase}; -use hotg_rune_core::{ElementType, Shape}; -use legion::{systems::CommandBuffer, world::SubWorld, Entity, Query}; -use proc_macro2::{Ident, Literal, Span, TokenStream}; -use quote::{quote, ToTokens}; - -use crate::{ - codegen::{CustomSection, File}, - lowering::{ - Inputs, Mimetype, Model, ModelFile, Name, Outputs, PipelineNode, - ProcBlock, Resource, ResourceData, ResourceOrString, Sink, SinkKind, - Source, Tensor, - }, - parse::ResourceType, -}; - -/// Generate the entire `lib.rs` file. -/// -/// FIXME: This should be split up into different phases for generating each -/// part in the `lib.rs` file. Some low hanging fruit are things like the -/// resources and models modules or the initializers for each pipeline node -/// because they can be done using a `#[legion::system(for_each)]`. -#[legion::system] -pub(crate) fn run( - cmd: &mut CommandBuffer, - world: &SubWorld, - sections: &mut Query<&CustomSection>, - models: &mut Query<(&Name, &Model, &Mimetype, &Inputs, &Outputs)>, - names: &mut Query<&Name>, - tensors: &mut Query<(Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)>, - tensor_by_ent: &mut Query<&Tensor>, - resources: &mut Query<(&Name, &Resource, Option<&ResourceData>)>, - capabilities: &mut Query<(&Name, &Source, &Outputs)>, - proc_blocks: &mut Query<(&Name, &ProcBlock)>, - outputs: &mut Query<(&Name, &Sink)>, - pipeline_nodes: &mut Query<( - Entity, - &Name, - Option<&Inputs>, - Option<&Outputs>, - &PipelineNode, - )>, -) { - let models: Vec<_> = models.iter(world).collect(); - let sections: Vec<_> = sections.iter(world).collect(); - let resources: Vec<_> = resources.iter(world).collect(); - let capabilities: Vec<_> = capabilities.iter(world).collect(); - let proc_blocks: Vec<_> = proc_blocks.iter(world).collect(); - let outputs: Vec<_> = outputs.iter(world).collect(); - let pipeline_nodes: Vec<_> = pipeline_nodes.iter(world).collect(); - let tensors: Vec<_> = tensors.iter(world).collect(); - - let lib_rs = generate_lib_rs( - §ions, - &models, - &resources, - &capabilities, - &proc_blocks, - &outputs, - &pipeline_nodes, - &tensors, - |ent| names.get(world, ent).ok(), - |ent| tensor_by_ent.get(world, ent).ok(), - ); - let file = File::new("lib.rs", lib_rs.to_string().into_bytes()); - - cmd.push((file,)); -} - -fn generate_lib_rs<'world>( - sections: &[&CustomSection], - models: &'world [( - &'world Name, - &'world Model, - &'world Mimetype, - &'world Inputs, - &'world Outputs, - )], - resources: &[(&Name, &Resource, Option<&ResourceData>)], - capabilities: &[(&Name, &Source, &Outputs)], - proc_blocks: &[(&Name, &ProcBlock)], - outputs: &[(&Name, &Sink)], - pipeline_nodes: &[Node<'_>], - tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], - mut get_name: impl FnMut(Entity) -> Option<&'world Name>, - mut get_tensor: impl FnMut(Entity) -> Option<&'world Tensor>, -) -> TokenStream { - let prelude = generate_prelude(); - let custom_sections = generate_custom_sections(sections); - let resources_module = generate_resources_module(resources); - let models_module = generate_models_module( - models.iter().map(|(n, m, ..)| (*n, *m)), - &mut get_name, - ); - let manifest = generate_manifest_function( - models, - capabilities, - proc_blocks, - outputs, - pipeline_nodes, - tensors, - &mut get_name, - &mut get_tensor, - ); - let call = generate_call_function(); - - quote! { - #prelude - #custom_sections - #resources_module - #models_module - #manifest - #call - } -} - -/// Generate a `manifest()` function that initializes the various nodes in -/// our pipeline then turns it into a closure that gets stored in the -/// `PIPELINE` static variable. -fn generate_manifest_function<'world, F, T>( - models: &[(&Name, &Model, &Mimetype, &Inputs, &Outputs)], - capabilities: &[(&Name, &Source, &Outputs)], - proc_blocks: &[(&Name, &ProcBlock)], - outputs: &[(&Name, &Sink)], - pipeline_nodes: &[Node<'_>], - tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], - get_name: &mut F, - get_tensor: &mut T, -) -> TokenStream -where - F: FnMut(Entity) -> Option<&'world Name>, - T: FnMut(Entity) -> Option<&'world Tensor>, -{ - let capabilities = - initialize_capabilities(capabilities, get_tensor, get_name); - let proc_blocks = initialize_proc_blocks(proc_blocks, get_name); - let models: TokenStream = models - .iter() - .map(|(n, m, mt, i, o)| { - initialize_model(n, m, mt, i, o, get_name, get_tensor) - }) - .collect(); - let outputs = initialize_outputs(outputs); - let pipeline = execute_pipeline(pipeline_nodes, tensors); - - quote! { - #[no_mangle] - pub extern "C" fn _manifest() -> i32 { - let _setup = hotg_runicos_base_wasm::SetupGuard::default(); - #capabilities - #proc_blocks - #models - #outputs - - let pipeline = move || { - let _guard = hotg_runicos_base_wasm::PipelineGuard::default(); - #pipeline - }; - - unsafe { - PIPELINE = Some(Box::new(pipeline)); - } - - 1 - } - } -} - -fn execute_pipeline( - pipeline_nodes: &[( - &Entity, - &Name, - Option<&Inputs>, - Option<&Outputs>, - &PipelineNode, - )], - tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], -) -> TokenStream { - let ExecutionOrder { - order, - tensor_names, - pipeline_nodes, - .. - } = ExecutionOrder::calculate(pipeline_nodes, tensors); - - order - .iter() - .map(|entity| { - execute_pipeline_node( - entity, - &pipeline_nodes, - &tensor_names, - tensors, - ) - }) - .collect() -} - -fn execute_pipeline_node( - node: &Entity, - pipeline_nodes: &HashMap< - Entity, - (&Name, Option<&Inputs>, Option<&Outputs>), - >, - tensor_names: &HashMap, - tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], -) -> TokenStream { - let (name, inputs, outputs) = pipeline_nodes - .get(node) - .copied() - .expect("This pipeline node always be present"); - - match (inputs, outputs) { - (Some(inputs), Some(outputs)) => execute_model_or_proc_block( - name, - inputs, - outputs, - tensor_names, - tensors, - ), - (None, Some(outputs)) => { - execute_capability(name, outputs, tensor_names, tensors) - }, - (Some(inputs), None) => execute_output(name, inputs, tensor_names), - (None, None) => { - unreachable!( - "The \"{}\" pipeline node should have inputs and/or outputs", - name - ) - }, - } -} - -fn execute_output( - name: &Name, - inputs: &Inputs, - tensor_names: &HashMap, -) -> TokenStream { - let name = Ident::new(name, Span::call_site()); - let inputs = input_bindings(&inputs.tensors, tensor_names); - - let msg = format!("Sending results to the \"{}\" output", name); - - quote! { - log::debug!(#msg); - #name.consume(#inputs); - } -} - -fn execute_model_or_proc_block( - name: &Name, - inputs: &Inputs, - outputs: &Outputs, - tensor_names: &HashMap, - tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], -) -> TokenStream { - let name = Ident::new(name, Span::call_site()); - let inputs = input_bindings(&inputs.tensors, tensor_names); - let output_types = tensor_types(&outputs.tensors, tensors); - let outputs = tensor_name_or_tuple(&outputs.tensors, tensor_names); - - let msg = format!("Executing \"{}\"", name); - - quote! { - log::debug!(#msg); - let #outputs: #output_types = #name.transform(#inputs); - } -} - -fn input_bindings( - tensors: &[Entity], - tensor_names: &HashMap, -) -> TokenStream { - let names: Vec<_> = tensors.iter().map(|t| &tensor_names[t]).collect(); - - match names.as_slice() { - [] => unreachable!("Expected 1 or more tensors"), - [tensor] => quote!(#tensor.clone()), - names => quote!((#( #names.clone() ),*)), - } -} - -fn tensor_types( - tensors: &[Entity], - all_tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], -) -> TokenStream { - let mut types = Vec::new(); - - for ent in tensors { - let (_, Tensor(shape), _, _) = all_tensors - .iter() - .copied() - .find(|(e, _, _, _)| ent == *e) - .unwrap(); - - types.push(shape_to_tensor_type(shape)); - } - - match types.as_slice() { - [single] => single.clone(), - many => quote!((#(#many),*)), - } -} - -fn shape_to_tensor_type(shape: &Shape) -> TokenStream { - let element_type = match shape.element_type() { - ElementType::U8 => quote!(u8), - ElementType::I8 => quote!(i8), - ElementType::U16 => quote!(u16), - ElementType::I16 => quote!(i16), - ElementType::U32 => quote!(u32), - ElementType::I32 => quote!(i32), - ElementType::F32 => quote!(f32), - ElementType::U64 => quote!(u64), - ElementType::I64 => quote!(i64), - ElementType::F64 => quote!(f64), - ElementType::String => quote!(alloc::borrow::Cow<'static, str>), - }; - quote!(Tensor<#element_type>) -} - -fn tensor_name_or_tuple( - tensors: &[Entity], - tensor_names: &HashMap, -) -> TokenStream { - let names: Vec<_> = tensors.iter().map(|t| &tensor_names[t]).collect(); - - match names.as_slice() { - [] => unreachable!("Expected 1 or more tensors"), - [tensor] => tensor.into_token_stream(), - names => quote!((#(#names),*)), - } -} - -fn execute_capability( - name: &Name, - outputs: &Outputs, - tensor_names: &HashMap, - tensors: &[(&Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)], -) -> TokenStream { - let name = Ident::new(name, Span::call_site()); - let output_types = tensor_types(&outputs.tensors, tensors); - let outputs = tensor_name_or_tuple(&outputs.tensors, tensor_names); - - let msg = format!("Reading data from \"{}\"", name); - - quote! { - log::debug!(#msg); - let #outputs: #output_types = #name.generate(); - } -} - -#[derive(Debug, Default)] -struct ExecutionOrder<'world> { - order: Vec, - tensor_names: HashMap, - // internal bookkeeping - visited_nodes: HashSet, - pipeline_nodes: HashMap< - Entity, - ( - &'world Name, - Option<&'world Inputs>, - Option<&'world Outputs>, - ), - >, - tensor_inputs: HashMap, -} - -type Node<'world> = ( - &'world Entity, - &'world Name, - Option<&'world Inputs>, - Option<&'world Outputs>, - &'world PipelineNode, -); - -impl<'world> ExecutionOrder<'world> { - /// Given a set of pipeline nodes, determine the order they should be - /// executed in and variable names for the various tensors involved. - /// - /// # Notes - /// - /// This assumes the pipeline nodes define a directed acyclic graph, and may - /// not return if it contains cycles. - /// - /// This does [a topological sort][topo] using a modified depth-first - /// search. - /// - /// [topo]: https://www.geeksforgeeks.org/topological-sorting/ - fn calculate( - pipeline_nodes: &'world [Node<'world>], - tensors: &'world [( - &'world Entity, - &'world Tensor, - Option<&'world Inputs>, - Option<&'world Outputs>, - )], - ) -> Self { - let mut order = ExecutionOrder { - order: Vec::new(), - tensor_names: HashMap::new(), - visited_nodes: HashSet::new(), - pipeline_nodes: pipeline_nodes - .iter() - .copied() - .map(|(ent, name, inputs, outputs, _)| { - (*ent, (name, inputs, outputs)) - }) - .collect(), - tensor_inputs: tensors - .iter() - .copied() - .map(|(ent, _, inputs, _)| { - ( - *ent, - inputs - .map(|i| i.tensors.as_slice()) - .unwrap_or_default(), - ) - }) - .collect(), - }; - - for (entity, ..) in pipeline_nodes.iter().copied() { - order.visit(*entity); - } - - order - } - - fn visit(&mut self, entity: Entity) { - if self.visited_nodes.contains(&entity) { - return; - } - - self.visited_nodes.insert(entity); - - let (name, inputs, outputs) = self.pipeline_nodes[&entity]; - - // We need to make sure all the inputs have been initialized first - if let Some(inputs) = inputs { - for input in &inputs.tensors { - let previous_nodes = - self.tensor_inputs.get(input).copied().expect( - "All tensors must have a node that created them", - ); - for &previous_node in previous_nodes { - self.visit(previous_node); - } - } - } - - // the pipeline node is executed - self.order.push(entity); - - // and now it's been executed, we can mark each of its outputs as - // available. - if let Some(outputs) = outputs { - for (i, tensor) in outputs.tensors.iter().enumerate() { - let tensor_name = format!("{}_{}", name, i); - self.tensor_names.insert( - *tensor, - Ident::new(&tensor_name, Span::call_site()), - ); - } - } - } -} - -fn initialize_outputs(outputs: &[(&Name, &Sink)]) -> TokenStream { - outputs - .iter() - .map(|(name, sink)| initialize_output(name, sink)) - .collect() -} - -fn initialize_output(name: &Name, sink: &Sink) -> TokenStream { - let name = Ident::new(name, Span::call_site()); - let type_name: TokenStream = sink_type_name(&sink.kind); - - quote! { - let mut #name = #type_name::default(); - } -} - -fn sink_type_name(kind: &SinkKind) -> TokenStream { - match kind { - SinkKind::Serial => quote!(hotg_runicos_base_wasm::Serial), - SinkKind::Tensor => quote!(hotg_runicos_base_wasm::TensorOutput), - SinkKind::Other(other) => { - unimplemented!("Unable to handle \"{}\" outputs", other) - }, - } -} - -fn initialize_model<'world, N, T>( - name: &Name, - model: &Model, - mimetype: &Mimetype, - inputs: &Inputs, - outputs: &Outputs, - get_name: &mut N, - get_tensor: &mut T, -) -> TokenStream -where - N: FnMut(Entity) -> Option<&'world Name>, - T: FnMut(Entity) -> Option<&'world Tensor>, -{ - let name = Ident::new(name, Span::call_site()); - - let path_to_model_bytes = match &model.model_file { - ModelFile::FromDisk(_) => quote!(crate::models::#name), - ModelFile::Resource(resource) => { - let resource_name = get_name(*resource) - .expect("We should always be able to get a resource's name"); - let resource_name = Ident::new(resource_name, Span::call_site()); - quote!(crate::resources::#resource_name) - }, - }; - - let input_descriptors: TokenStream = - tensor_descriptors(&inputs.tensors, get_tensor); - let output_descriptors: TokenStream = - tensor_descriptors(&outputs.tensors, get_tensor); - - let mimetype = mimetype.as_ref(); - - quote! { - let mut #name = hotg_runicos_base_wasm::Model::load( - #mimetype, - &#path_to_model_bytes, - #input_descriptors, - #output_descriptors, - ); - } -} - -fn tensor_descriptors<'world, T>( - tensors: &[Entity], - get_tensor: &mut T, -) -> TokenStream -where - T: FnMut(Entity) -> Option<&'world Tensor>, -{ - let inputs = tensors - .iter() - .map(|&ent| { - get_tensor(ent).expect("All tensors should have been allocated") - }) - .map(|t| shape_to_tokens(&t.0)); - quote! { &[#(#inputs),*] } -} - -fn element_type_to_tokens(element_type: ElementType) -> TokenStream { - let name = match element_type { - ElementType::U8 => "U8", - ElementType::I8 => "I8", - ElementType::U16 => "U16", - ElementType::I16 => "I16", - ElementType::U32 => "U32", - ElementType::F32 => "F32", - ElementType::I32 => "I32", - ElementType::U64 => "U64", - ElementType::F64 => "F64", - ElementType::I64 => "I64", - ElementType::String => "String", - }; - let ident = Ident::new(name, Span::call_site()); - quote!(hotg_rune_core::ElementType::#ident) -} - -fn shape_to_tokens(shape: &Shape<'_>) -> TokenStream { - let element_type = element_type_to_tokens(shape.element_type()); - let dimensions = shape.dimensions(); - - quote! { - hotg_rune_core::Shape::new( - #element_type, - [ #(#dimensions),* ].as_ref(), - ) - } -} - -fn initialize_proc_blocks<'world, N>( - proc_blocks: &[(&Name, &ProcBlock)], - get_name: &mut N, -) -> TokenStream -where - N: FnMut(Entity) -> Option<&'world Name>, -{ - proc_blocks - .iter() - .copied() - .map(|(name, proc_block)| { - initialize_proc_block(name, proc_block, get_name) - }) - .collect() -} - -fn initialize_proc_block<'world, N>( - name: &Name, - proc_block: &ProcBlock, - get_name: &mut N, -) -> TokenStream -where - N: FnMut(Entity) -> Option<&'world Name>, -{ - let ty = proc_block_type(proc_block); - - let name = Ident::new(name, Span::call_site()); - let setters = proc_block.parameters.iter().map(|(key, value)| { - let value = proc_block_argument_to_tokens(value, get_name); - let setter = format!("set_{}", key).replace("-", "_"); - let setter = Ident::new(&setter, Span::call_site()); - let error_message = - format!("Unable to set {}'s \"{}\" to {}", name, key, value); - quote! { - #name.#setter(#value).expect(#error_message); - } - }); - - quote! { - let mut #name = #ty::default(); - #( #setters )* - } -} - -fn proc_block_type(proc_block: &ProcBlock) -> TokenStream { - let module_name = proc_block.name().to_snake_case(); - let type_name = module_name.to_upper_camel_case(); - - let module_name = Ident::new(&module_name, Span::call_site()); - let type_name = Ident::new(&type_name, Span::call_site()); - - quote!(#module_name::#type_name) -} - -fn initialize_capabilities<'world, T, N>( - capabilities: &[(&Name, &Source, &Outputs)], - get_tensor: &mut T, - get_name: &mut N, -) -> TokenStream -where - T: FnMut(Entity) -> Option<&'world Tensor>, - N: FnMut(Entity) -> Option<&'world Name>, -{ - capabilities - .iter() - .copied() - .map(|(name, source, outputs)| { - initialize_capability(name, source, outputs, get_tensor, get_name) - }) - .collect() -} - -fn initialize_capability<'world, T, N>( - name: &Name, - source: &Source, - outputs: &Outputs, - get_tensor: &mut T, - get_name: &mut N, -) -> TokenStream -where - T: FnMut(Entity) -> Option<&'world Tensor>, - N: FnMut(Entity) -> Option<&'world Name>, -{ - let capability_type = match source.kind.as_capability_name() { - Some(name) => { - let name = Ident::new(name, Span::call_site()); - quote!(hotg_rune_core::capabilities::#name) - }, - None => unimplemented!( - "Unable to generate code for the \"{}\" capability type", - source.kind - ), - }; - - let output_tensor = match outputs.tensors.as_slice() { - [tensor] => get_tensor(*tensor).unwrap(), - _ => unreachable!("Capabilities should only have one output"), - }; - let shape = shape_to_tokens(&output_tensor.0); - - let name = Ident::new(name, Span::call_site()); - let setters = source.parameters.iter().map(|(key, value)| { - let key = key.replace("-", "_"); - let value = capability_argument_to_tokens(value, get_name); - quote! { - #name.set_parameter(#key, #value); - } - }); - - quote! { - let mut #name = hotg_runicos_base_wasm::Capability::new(#capability_type, #shape); - #( #setters )* - } -} - -fn proc_block_argument_to_tokens<'world, F>( - value: &ResourceOrString, - get_name: &mut F, -) -> TokenStream -where - F: FnMut(Entity) -> Option<&'world Name>, -{ - match value { - ResourceOrString::String(s) => quote!(#s), - ResourceOrString::Resource(r) => { - let name = get_name(*r).unwrap(); - let resource_name = Ident::new(&name, Span::call_site()); - quote!(&*crate::resources::#resource_name) - }, - } -} - -/// Take a [`ResourceOrString`] and turn it into an `impl Into` -/// expression so it can be passed to a capability. -/// -/// Note: this *could* be merged with [`proc_block_argument_to_tokens`] if -/// capabilities accepted strings and the image capability didn't need our -/// `hotg_rune_core::ImageFormat` hack wher `@` lets you pass in arbitrary Rust -/// expressions. -fn capability_argument_to_tokens<'world, F>( - value: &ResourceOrString, - get_name: &mut F, -) -> TokenStream -where - F: FnMut(Entity) -> Option<&'world Name>, -{ - match value { - ResourceOrString::String(s) => { - if let Some(stripped) = s.strip_prefix('@') { - stripped.parse::().unwrap_or_else(|e| { - let msg = format!( - "Unable to parse \"{}\" as a Rust expression: {}", - stripped, e - ); - quote!(compile_error!(#msg)) - }) - } else { - quote! { - #s - .parse::() - .unwrap_or_else(|_| { panic!( "Unable to parse \"{}\" as a number", #s); }) - } - } - }, - ResourceOrString::Resource(r) => { - let name = get_name(*r).unwrap(); - let resource_name = Ident::new(&name, Span::call_site()); - quote!(&*crate::resources::#resource_name) - }, - } -} - -/// Imports and miscellaneous attributes added to the top of the file. -fn generate_prelude() -> TokenStream { - quote! { - //! Automatically generated by Rune. DO NOT EDIT! - - #![no_std] - #![feature(alloc_error_handler)] - #![allow(warnings)] - - extern crate alloc; - - #[macro_use] - extern crate lazy_static; - - use alloc::boxed::Box; - use hotg_rune_core::PixelFormat; - use hotg_rune_proc_blocks::*; - - static mut PIPELINE: Option> = None; - } -} - -/// The `call()` function - a simple function which invokes the `PIPELINE` -/// constructed by [`generate_manifest_function()`]. -fn generate_call_function() -> TokenStream { - quote! { - #[no_mangle] - pub extern "C" fn _call( - _capability_type: i32, - _input_type: i32, - _capability_idx: i32, - ) -> i32 { - unsafe { - let pipeline = PIPELINE.as_mut() - .expect("The rune hasn't been initialized"); - pipeline(); - - 0 - } - } - } -} - -/// Generate WebAssembly custom sections which are used to embed metadata in -/// the compiled Rune. -fn generate_custom_sections(sections: &[&CustomSection]) -> TokenStream { - let sections = sections - .iter() - .enumerate() - .map(|(i, section)| generate_custom_section(i, *section)); - - quote! { - /// Custom sections embedded in the Rune that can be inspected later. - /// - /// # Note - /// - /// These sections need to be at the top level to make sure the linker - /// won't remove them during its "gc sections" pass, but we also don't - /// want to pollute the top-level namespace so we put it inside an - /// unnamed constant. - const _: () = { - #( #sections )* - }; - } -} - -/// Generate the declaration for a [`CustomSection`], appending a unique number -/// to help avoid duplicates when you've got multiple [`CustomSection`]s with -/// the same name. -fn generate_custom_section( - section_number: usize, - s: &CustomSection, -) -> TokenStream { - let unique_ident = format!("{}_{}", s.identifier(), section_number); - let ident = Ident::new(&unique_ident, Span::call_site()); - let section_name = &s.section_name; - let data = Literal::byte_string(&s.value); - let len = s.value.len(); - - quote! { - #[link_section = #section_name] - static #ident: [u8; #len] = *#data; - } -} - -fn generate_resources_module( - resources: &[(&Name, &Resource, Option<&ResourceData>)], -) -> TokenStream { - let initializers = resources - .iter() - .copied() - .map(|(name, res, data)| resource_initializer(name, res, data)); - - quote! { - /// Lazily loaded accessors for all resources used by this Rune. - mod resources { - lazy_static::lazy_static! { - #(#initializers)* - } - } - } -} - -fn resource_initializer( - name: &Name, - res: &Resource, - data: Option<&ResourceData>, -) -> TokenStream { - let name = name.as_str(); - - // First we try to read the resource using the runtime, returning a - // Result, _> - let maybe_bytes = quote! { - hotg_runicos_base_wasm::Resource::read_to_end(#name) - }; - - // We then take the Result and unwrap it, either falling back to a default - // value (provided in the Runefile) or blowing up - let bytes = match data { - Some(default_value) => { - let default_value = Literal::byte_string(default_value); - quote!(#maybe_bytes.unwrap_or_else(|_| #default_value.as_ref().into())) - }, - None => { - let error_message = - format!("Unable to read the \"{}\" resource", name); - quote!(#maybe_bytes.expect(#error_message)) - }, - }; - - let ident = Ident::new(name, Span::call_site()); - - // And now we can initialize our "static ref" - match res.ty { - ResourceType::String => { - let error_message = - format!("The \"{}\" resource isn't valid UTF-8", name); - - quote! { - pub(crate) static ref #ident: alloc::string::String = { - let bytes = #bytes; - core::str::from_utf8(&bytes).expect(#error_message).into() - }; - } - }, - ResourceType::Binary => { - quote! { - pub(crate) static ref #ident: alloc::vec::Vec = #bytes.to_vec(); - } - }, - } -} - -fn generate_models_module<'world, N, M>( - models: M, - get_name: &mut N, -) -> TokenStream -where - M: Iterator, - N: FnMut(Entity) -> Option<&'world Name>, -{ - let initializers = - models.map(|(name, model)| model_initializer(name, model, get_name)); - - quote! { - /// Lazily loaded accessors for all models used by this Rune. - mod models { - lazy_static::lazy_static! { - #(#initializers)* - } - } - } -} - -fn model_initializer<'world, N>( - name: &Name, - model: &Model, - get_name: &mut N, -) -> TokenStream -where - N: FnMut(Entity) -> Option<&'world Name>, -{ - let name = Ident::new(name, Span::call_site()); - - match &model.model_file { - ModelFile::FromDisk(_) => { - let path = format!("models/{}", name); - - quote! { - pub(crate) static ref #name: &'static [u8] = include_bytes!(#path); - } - }, - ModelFile::Resource(resource) => { - let resource_name = get_name(*resource).unwrap(); - let resource_name = Ident::new(resource_name, Span::call_site()); - - quote! { - pub(crate) static ref #name: &'static [u8] = crate::resources::#resource_name.as_ref(); - } - }, - } -} - -#[cfg(test)] -mod tests { - use std::{ - io::{Read, Write}, - process::{Command, Stdio}, - }; - - use legion::{IntoQuery, Resources, World}; - - use super::*; - - fn rustfmt(tokens: TokenStream) -> String { - let mut child = Command::new("rustfmt") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn() - .unwrap(); - - // Note: We need to wrap the fragment in a function so it'll parse - let mut stdin = child.stdin.take().unwrap(); - writeln!(stdin, "fn main() {{").unwrap(); - writeln!(stdin, "{}", tokens).unwrap(); - writeln!(stdin, "}}").unwrap(); - stdin.flush().unwrap(); - drop(stdin); - - let mut stdout = child.stdout.take().unwrap(); - let mut pretty = String::new(); - stdout.read_to_string(&mut pretty).unwrap(); - - let opening_curly = pretty.find('{').unwrap(); - let closing_curly = pretty.rfind('}').unwrap(); - - pretty[opening_curly + 1..closing_curly].trim().to_string() - } - - macro_rules! assert_quote_eq { - ($left:expr, $right:expr) => {{ - let left = $left.to_string(); - let right = $right.to_string(); - - if left != right { - let pretty_left = rustfmt($left); - let pretty_right = rustfmt($right); - assert_eq!(pretty_left, pretty_right); - assert_eq!(left, right); - } - }}; - } - - #[test] - fn custom_section() { - let section = CustomSection::new(".name", b"hello world".as_ref()); - let should_be = quote! { - #[link_section = ".name"] - static name_42: [u8; 11usize] = *b"hello world"; - }; - - let got = generate_custom_section(42, §ion); - - assert_quote_eq!(got, should_be); - } - - #[test] - fn simple_linear_execution_order() { - let mut world = World::default(); - let mut resources = Resources::default(); - let mut cmd = CommandBuffer::new(&world); - // manually add the first node - let first_output = cmd.push((Tensor("f32[1]".parse().unwrap()),)); - let first = cmd.push(( - Name::from("first"), - Outputs { - tensors: vec![first_output], - }, - PipelineNode, - )); - // add the second node - let second_output = cmd.push((Tensor("f32[1]".parse().unwrap()),)); - let second = cmd.push(( - Name::from("second"), - Inputs { - tensors: vec![first_output], - }, - Outputs { - tensors: vec![second_output], - }, - PipelineNode, - )); - // Add the third node - let third = cmd.push(( - Name::from("third"), - Inputs { - tensors: vec![second_output], - }, - PipelineNode, - )); - cmd.flush(&mut world, &mut resources); - - let pipeline_nodes: Vec<_> = <( - Entity, - &Name, - Option<&Inputs>, - Option<&Outputs>, - &PipelineNode, - )>::query() - .iter(&world) - .collect(); - let tensors: Vec<_> = - <(Entity, &Tensor, Option<&Inputs>, Option<&Outputs>)>::query() - .iter(&world) - .collect(); - - let ExecutionOrder { - order, - tensor_names, - .. - } = ExecutionOrder::calculate(&pipeline_nodes, &tensors); - - let order_should_be = vec![first, second, third]; - assert_eq!(order, order_should_be); - let tensor_names_should_be: HashMap<_, _> = vec![ - (first_output, Ident::new("first_0", Span::call_site())), - (second_output, Ident::new("second_0", Span::call_site())), - ] - .into_iter() - .collect(); - assert_eq!(tensor_names, tensor_names_should_be); - } - - #[test] - fn execute_a_capability() { - let mut world = World::default(); - let mut resources = Resources::default(); - let mut cmd = CommandBuffer::new(&world); - let first_output_tensor = Tensor("f32[1]".parse().unwrap()); - let first_output = cmd.push((first_output_tensor.clone(),)); - let name = Name::from("first"); - let outputs = Outputs { - tensors: vec![first_output], - }; - cmd.flush(&mut world, &mut resources); - let tensor_names: HashMap<_, _> = - vec![(first_output, Ident::new("first_0", Span::call_site()))] - .into_iter() - .collect(); - let tensors = &[(&first_output, &first_output_tensor, None, None)]; - - let got = execute_capability(&name, &outputs, &tensor_names, tensors); - - let should_be = quote! { - log::debug!("Reading data from \"first\""); - let first_0: Tensor = first.generate(); - }; - assert_quote_eq!(got, should_be); - } - - #[test] - fn execute_model() { - let mut world = World::default(); - let mut resources = Resources::default(); - let mut cmd = CommandBuffer::new(&world); - let model_output_tensor = Tensor("f32[1]".parse().unwrap()); - let model_output = cmd.push((model_output_tensor.clone(),)); - let model_input_tensor = Tensor("u8[1, 1, 1]".parse().unwrap()); - let model_input = cmd.push((model_input_tensor.clone(),)); - let name = Name::from("model"); - cmd.flush(&mut world, &mut resources); - let inputs = Inputs { - tensors: vec![model_input], - }; - let outputs = Outputs { - tensors: vec![model_output], - }; - let tensor_names: HashMap<_, _> = vec![ - (model_output, Ident::new("model_output", Span::call_site())), - (model_input, Ident::new("model_input", Span::call_site())), - ] - .into_iter() - .collect(); - let tensors = &[ - (&model_output, &model_output_tensor, None, None), - (&model_input, &model_input_tensor, None, None), - ]; - - let got = execute_model_or_proc_block( - &name, - &inputs, - &outputs, - &tensor_names, - tensors, - ); - - let should_be = quote! { - log::debug!("Executing \"model\""); - let model_output: Tensor = model.transform(model_input.clone()); - }; - assert_quote_eq!(got, should_be); - } - - #[test] - fn consume_multiple_outputs() { - let mut world = World::default(); - let mut resources = Resources::default(); - let mut cmd = CommandBuffer::new(&world); - let first_input_tensor = - cmd.push((Tensor("u8[1, 1, 1]".parse().unwrap()),)); - let second_input_tensor = - cmd.push((Tensor("f32[128]".parse().unwrap()),)); - let name = Name::from("serial"); - cmd.flush(&mut world, &mut resources); - let inputs = Inputs { - tensors: vec![first_input_tensor, second_input_tensor], - }; - let tensor_names: HashMap<_, _> = vec![ - ( - first_input_tensor, - Ident::new("first_input", Span::call_site()), - ), - ( - second_input_tensor, - Ident::new("second_input", Span::call_site()), - ), - ] - .into_iter() - .collect(); - - let got = execute_output(&name, &inputs, &tensor_names); - - let should_be = quote! { - log::debug!("Sending results to the \"serial\" output"); - serial.consume((first_input.clone(), second_input.clone())); - }; - assert_quote_eq!(got, should_be); - } - - #[test] - fn tensor_shapes_as_rust_types() { - let inputs = vec![ - ("f32[1]", quote!(Tensor)), - ("u8[1, 2, 3, 4]", quote!(Tensor)), - ("utf8[42]", quote!(Tensor>)), - ]; - - for (shape, should_be) in inputs { - let shape: Shape<'_> = shape.parse().unwrap(); - let got = shape_to_tensor_type(&shape); - assert_eq!( - got.to_string().replace(" ", ""), - should_be.to_string().replace(" ", "") - ); - } - } -} diff --git a/crates/compiler/src/codegen/generate_model_files.rs b/crates/compiler/src/codegen/generate_model_files.rs deleted file mode 100644 index ac3a608ddf..0000000000 --- a/crates/compiler/src/codegen/generate_model_files.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::path::Path; - -use legion::systems::CommandBuffer; - -use crate::{ - codegen::File, - lowering::{ModelData, Name}, -}; - -/// Create a [`File`] for each model with associated [`ModelData`] and put it in -/// the `models/` directory. -#[legion::system(for_each)] -pub(crate) fn run(cmd: &mut CommandBuffer, name: &Name, data: &ModelData) { - let path = Path::new("models").join(name.as_str()); - let file = File::new(path, data.0.clone()); - cmd.push((file,)); -} diff --git a/crates/compiler/src/codegen/generate_resource_section.rs b/crates/compiler/src/codegen/generate_resource_section.rs deleted file mode 100644 index 9e2469e44e..0000000000 --- a/crates/compiler/src/codegen/generate_resource_section.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::convert::TryFrom; - -use legion::{systems::CommandBuffer, world::SubWorld, Entity, Query}; - -use crate::{ - codegen::{CustomSection, RESOURCE_CUSTOM_SECTION}, - lowering::{Name, ResourceData}, -}; - -/// Generate [`CustomSection`]s that embed each resource's default value in -/// the Rune. -#[legion::system] -pub(crate) fn run( - cmd: &mut CommandBuffer, - world: &SubWorld, - resources: &mut Query<(Entity, &Name, &ResourceData)>, -) { - resources.for_each(world, |(&entity, name, data)| { - cmd.add_component(entity, inline_resource(name, data)); - }); -} - -pub(crate) fn inline_resource( - name: &Name, - data: &ResourceData, -) -> CustomSection { - let name_len = u32::try_from(name.len()).unwrap(); - let data_len = u32::try_from(data.len()).unwrap(); - let buffer_length = std::mem::size_of_val(&name_len) - + name.len() - + std::mem::size_of_val(&data_len) - + data.len(); - let mut buffer = Vec::with_capacity(buffer_length); - - buffer.extend(name_len.to_be_bytes()); - buffer.extend_from_slice(name.as_bytes()); - buffer.extend(data_len.to_be_bytes()); - buffer.extend_from_slice(data); - - CustomSection { - section_name: RESOURCE_CUSTOM_SECTION.to_string(), - value: buffer.into(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn create_a_custom_section_for_a_resource() { - let data = ResourceData::from(&b"Hello, World!"[..]); - let name = Name::from("my_resource"); - - let CustomSection { - section_name, - value, - } = inline_resource(&name, &data); - - assert_eq!(section_name, RESOURCE_CUSTOM_SECTION); - let (resource_name, resource_data, rest) = - hotg_rune_core::decode_inline_resource(&value).unwrap(); - assert_eq!(resource_name, name.as_str()); - assert_eq!(resource_data, data.as_ref()); - assert!(rest.is_empty()); - } -} diff --git a/crates/compiler/src/codegen/generate_rune_graph_section.rs b/crates/compiler/src/codegen/generate_rune_graph_section.rs deleted file mode 100644 index b1801b6a78..0000000000 --- a/crates/compiler/src/codegen/generate_rune_graph_section.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::collections::HashMap; - -use indexmap::IndexMap; -use legion::{ - serialize::{Canon, CustomEntitySerializer}, - systems::CommandBuffer, - world::SubWorld, - Entity, Query, -}; - -use super::{CapabilitySummary, RuneSummary}; -use crate::{ - codegen::{ - Codegen, CustomSection, ModelSummary, OutputSummary, ProcBlockSummary, - RuneGraph, TensorId, - }, - lowering::{ - self, Inputs, Model, ModelFile, Name, Outputs, ProcBlock, Resource, - Sink, Source, Tensor, - }, - parse::{ResourceName, ResourceOrString}, - BuildContext, -}; - -pub(crate) fn rune_graph_section(_db: &dyn Codegen) -> CustomSection { todo!() } - -/// Generate an abbreviated [`RuneGraph`]. -#[legion::system] -pub(crate) fn run( - cmd: &mut CommandBuffer, - world: &SubWorld, - #[resource] ctx: &BuildContext, - capabilities: &mut Query<(&Name, &Source, &Outputs)>, - tensors: &mut Query<(Entity, &Tensor)>, - models: &mut Query<(&Name, &Model, &Inputs, &Outputs)>, - proc_blocks: &mut Query<(&Name, &ProcBlock, &Inputs, &Outputs)>, - outputs: &mut Query<(&Name, &Sink, &Inputs)>, - resources: &mut Query<(&Name, &Resource)>, -) { - let canon = Canon::default(); - let mut resource_name = |ent: Entity| { - resources - .get(world, ent) - .map(|(name, _)| ResourceName(name.to_string())) - .unwrap() - }; - - let graph = RuneGraph { - rune: rune_summary(ctx), - capabilities: capabilities - .iter(world) - .map(|(n, s, o)| { - capability_summary(n, s, o, &canon, &mut resource_name) - }) - .collect(), - models: models - .iter(world) - .map(|(n, m, i, o)| { - model_summary(n, m, i, o, &mut resource_name, &canon) - }) - .collect(), - proc_blocks: proc_blocks - .iter(world) - .map(|(n, p, i, o)| { - proc_block_summary(n, p, i, o, &canon, &mut resource_name) - }) - .collect(), - outputs: outputs - .iter(world) - .map(|(n, s, i)| { - output_summary(n, s, i, &canon, &mut resource_name) - }) - .collect(), - resources: resources - .iter(world) - .map(|(name, res)| (name.clone(), res.clone())) - .collect(), - tensors: tensors - .iter(world) - .map(|(ent, t)| { - (canon.to_serialized(*ent).to_string().into(), t.0.clone()) - }) - .collect(), - }; - - let graph_section = graph - .as_custom_section() - .expect("We should always be able to serialize to JSON"); - cmd.push((graph, graph_section)); -} - -fn rune_summary(ctx: &BuildContext) -> RuneSummary { - RuneSummary { - name: ctx.name.clone(), - } -} - -fn tensor_shapes(tensors: &[Entity], get_tensor: &Canon) -> Vec { - tensors - .iter() - .map(|&ent| get_tensor.to_serialized(ent)) - .map(|t| TensorId(t.to_string())) - .collect() -} - -fn capability_summary( - name: &Name, - source: &Source, - outputs: &Outputs, - get_tensor: &Canon, - mut resource_name: impl FnMut(Entity) -> ResourceName, -) -> (Name, CapabilitySummary) { - let summary = CapabilitySummary { - kind: source.kind.clone(), - args: convert_args(&source.parameters, &mut resource_name), - outputs: tensor_shapes(&outputs.tensors, get_tensor), - }; - - (name.clone(), summary) -} - -fn model_summary( - name: &Name, - model: &Model, - inputs: &Inputs, - outputs: &Outputs, - mut resources: impl FnMut(Entity) -> ResourceName, - get_tensor: &Canon, -) -> (Name, ModelSummary) { - let file = match &model.model_file { - ModelFile::FromDisk(path) => { - ResourceOrString::String(path.display().to_string()) - }, - ModelFile::Resource(entity) => { - ResourceOrString::Resource(resources(*entity)) - }, - }; - - let summary = ModelSummary { - file, - args: convert_args(&model.args, resources), - inputs: tensor_shapes(&inputs.tensors, get_tensor), - outputs: tensor_shapes(&outputs.tensors, get_tensor), - }; - - (name.clone(), summary) -} - -fn proc_block_summary( - name: &Name, - proc_block: &ProcBlock, - inputs: &Inputs, - outputs: &Outputs, - get_tensor: &Canon, - mut resource_name: impl FnMut(Entity) -> ResourceName, -) -> (Name, ProcBlockSummary) { - let summary = ProcBlockSummary { - path: proc_block.path.clone(), - args: proc_block - .parameters - .iter() - .map(|(key, value)| { - let value = match value { - lowering::ResourceOrString::String(s) => { - ResourceOrString::String(s.clone()) - }, - lowering::ResourceOrString::Resource(ent) => { - ResourceOrString::Resource(resource_name(*ent)) - }, - }; - (key.clone(), value) - }) - .collect(), - inputs: tensor_shapes(&inputs.tensors, get_tensor), - outputs: tensor_shapes(&outputs.tensors, get_tensor), - }; - - (name.clone(), summary) -} - -fn output_summary( - name: &Name, - sink: &Sink, - inputs: &Inputs, - get_tensor: &Canon, - mut get_resources: impl FnMut(Entity) -> ResourceName, -) -> (Name, OutputSummary) { - let summary = OutputSummary { - kind: sink.kind.clone(), - args: convert_args(&sink.args, &mut get_resources), - inputs: tensor_shapes(&inputs.tensors, get_tensor), - }; - - (name.clone(), summary) -} - -fn convert_args( - args: &IndexMap, - mut resources: impl FnMut(Entity) -> ResourceName, -) -> HashMap { - let mut converted = HashMap::new(); - - for (key, value) in args { - let value = match value { - lowering::ResourceOrString::String(s) => { - ResourceOrString::String(s.clone()) - }, - lowering::ResourceOrString::Resource(r) => { - ResourceOrString::Resource(resources(*r)) - }, - }; - - converted.insert(key.clone(), value); - } - - converted -} diff --git a/crates/compiler/src/codegen/generate_rust_toolchain_toml.rs b/crates/compiler/src/codegen/generate_rust_toolchain_toml.rs deleted file mode 100644 index 92ee33500e..0000000000 --- a/crates/compiler/src/codegen/generate_rust_toolchain_toml.rs +++ /dev/null @@ -1,13 +0,0 @@ -use legion::systems::CommandBuffer; - -use crate::codegen::File; - -#[legion::system] -pub(crate) fn run(cmd: &mut CommandBuffer) { - let rust_toolchain = crate::rust_toolchain(); - let contents = toml::to_vec(&rust_toolchain) - .expect("We can always serialize a hard-coded TOML object"); - let file = File::new("rust-toolchain.toml", contents); - - cmd.push((file,)); -} diff --git a/crates/compiler/src/codegen/generate_version_section.rs b/crates/compiler/src/codegen/generate_version_section.rs deleted file mode 100644 index 94be072faa..0000000000 --- a/crates/compiler/src/codegen/generate_version_section.rs +++ /dev/null @@ -1,19 +0,0 @@ -use legion::systems::CommandBuffer; - -use crate::{codegen::CustomSection, BuildContext}; - -/// Embed a [`crate::codegen::RuneVersion`] in the Rune as a [`CustomSection`]. -#[legion::system] -pub(crate) fn run(cmd: &mut CommandBuffer, #[resource] ctx: &BuildContext) { - if let Some(components) = version_section(ctx) { - cmd.push((components,)); - } -} - -pub(crate) fn version_section(ctx: &BuildContext) -> Option { - ctx.rune_version.as_ref().map(|version| { - version - .as_custom_section() - .expect("We should always be able to serialize to JSON") - }) -} diff --git a/crates/compiler/src/codegen/inputs.rs b/crates/compiler/src/codegen/inputs.rs deleted file mode 100644 index c27af7559b..0000000000 --- a/crates/compiler/src/codegen/inputs.rs +++ /dev/null @@ -1,57 +0,0 @@ -use im::Vector; - -use crate::{ - inputs::Inputs, - lowering::{Model, ModelData, Name, ProcBlock, Resource, ResourceData}, -}; - -#[salsa::query_group(CodegenInputsGroup)] -pub trait CodegenInputs: Inputs { - #[salsa::input] - fn node_inputs(&self, name: Name) -> crate::lowering::Inputs; - #[salsa::input] - fn node_outputs(&self, name: Name) -> crate::lowering::Outputs; - - #[salsa::input] - fn resource_names(&self) -> Vector; - #[salsa::input] - fn resource_info(&self, name: Name) -> Vector; - #[salsa::input] - fn resource_data(&self, name: Name) -> ResourceData; - fn all_resource_data(&self) -> Vector<(Name, ResourceData)>; - - #[salsa::input] - fn proc_block_names(&self) -> Vector; - #[salsa::input] - fn proc_block_info(&self, name: Name) -> ProcBlock; - fn all_proc_blocks(&self) -> Vector; - - #[salsa::input] - fn model_names(&self) -> Vector; - #[salsa::input] - fn model_info(&self, name: Name) -> Model; - #[salsa::input] - fn model_data(&self, name: Name) -> ModelData; - fn all_model_data(&self) -> Vector<(Name, ModelData)>; -} - -fn all_model_data(db: &dyn CodegenInputs) -> Vector<(Name, ModelData)> { - db.model_names() - .into_iter() - .map(|name| (name.clone(), db.model_data(name))) - .collect() -} - -fn all_resource_data(db: &dyn CodegenInputs) -> Vector<(Name, ResourceData)> { - db.resource_names() - .into_iter() - .map(|name| (name.clone(), db.resource_data(name))) - .collect() -} - -fn all_proc_blocks(db: &dyn CodegenInputs) -> Vector { - db.proc_block_names() - .into_iter() - .map(|name| db.proc_block_info(name)) - .collect() -} diff --git a/crates/compiler/src/codegen/mod.rs b/crates/compiler/src/codegen/mod.rs index 22ebf9a12e..68c159aa84 100644 --- a/crates/compiler/src/codegen/mod.rs +++ b/crates/compiler/src/codegen/mod.rs @@ -1,149 +1,3 @@ -//! The code generation phase. -//! -//! This takes the parsed and analysed Rune and generates all the necessary -//! files to make a Rust project. +mod query; -mod compile_generated_project; -mod components; -mod generate_cargo_config; -mod generate_cargo_toml; -mod generate_lib_rs; -mod generate_model_files; -mod generate_resource_section; -mod generate_rune_graph_section; -mod generate_rust_toolchain_toml; -mod generate_version_section; -pub(crate) mod inputs; - -use std::path::Path; - -pub use components::*; -use im::Vector; -use legion::Registry; - -use crate::{ - codegen::{ - generate_rune_graph_section::rune_graph_section, inputs::CodegenInputs, - }, - lowering::{Name, ResourceData}, - phases::Phase, - serialize::RegistryExt, -}; - -pub fn phase() -> Phase { - Phase::new() - .and_then(generate_rust_toolchain_toml::run_system) - .and_then(generate_cargo_config::run_system) - .and_then(generate_cargo_toml::run_system) - .and_then(generate_model_files::run_system) - .and_then(generate_resource_section::run_system) - .and_then(generate_version_section::run_system) - .and_then(generate_rune_graph_section::run_system) - .and_then(generate_lib_rs::run_system) - .and_then(compile_generated_project::run_system) -} - -pub(crate) fn register_components(registry: &mut Registry) { - registry - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::(); -} - -#[salsa::query_group(CodegenGroup)] -pub trait Codegen: CodegenInputs { - fn rust_toolchain_toml(&self) -> File; - fn cargo_config(&self) -> File; - fn cargo_toml(&self) -> File; - fn model_files(&self) -> Vector; - - fn resource_section(&self, name: Name, data: ResourceData) - -> CustomSection; - fn resource_sections(&self) -> Vector; - fn version_section(&self) -> Option; - fn rune_graph_section(&self) -> CustomSection; - fn lib_rs(&self) -> File; - - fn custom_sections(&self) -> Vector; - - fn files(&self) -> Vector; -} - -fn rust_toolchain_toml(_: &dyn Codegen) -> File { - let rust_toolchain = crate::rust_toolchain(); - let contents = toml::to_vec(&rust_toolchain) - .expect("We can always serialize a hard-coded TOML object"); - - File::new("rust-toolchain.toml", contents) -} - -fn cargo_config(db: &dyn Codegen) -> File { - let ctx = db.build_context(); - generate_cargo_config::generate_config(ctx.optimized) -} - -fn cargo_toml(db: &dyn Codegen) -> File { - let features = db.feature_flags(); - let proc_blocks = db.all_proc_blocks(); - let ctx = db.build_context(); - - generate_cargo_toml::generate(&features, proc_blocks.iter(), &ctx) -} - -fn model_files(db: &dyn Codegen) -> Vector { - let mut files = Vector::new(); - - for (name, data) in db.all_model_data() { - let path = Path::new("models").join(name.as_str()); - let file = File::new(path, data.0.clone()); - files.push_back(file); - } - - files -} - -fn resource_section( - _: &dyn Codegen, - name: Name, - data: ResourceData, -) -> CustomSection { - generate_resource_section::inline_resource(&name, &data) -} - -fn resource_sections(db: &dyn Codegen) -> Vector { - db.all_resource_data() - .into_iter() - .map(|(name, data)| db.resource_section(name, data)) - .collect() -} - -fn version_section(db: &dyn Codegen) -> Option { - let ctx = db.build_context(); - generate_version_section::version_section(&ctx) -} - -fn lib_rs(_db: &dyn Codegen) -> File { todo!() } - -fn custom_sections(db: &dyn Codegen) -> Vector { - let mut sections = Vector::new(); - - sections.push_back(db.rune_graph_section()); - if let Some(version) = db.version_section() { - sections.push_back(version); - } - sections.extend(db.resource_sections()); - - sections -} - -fn files(db: &dyn Codegen) -> Vector { - let mut files = Vector::new(); - - files.push_back(db.rust_toolchain_toml()); - files.push_back(db.cargo_config()); - files.push_back(db.cargo_toml()); - files.extend(db.model_files()); - - files -} +pub use self::query::{Codegen, CodegenStorage}; diff --git a/crates/compiler-2/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs similarity index 100% rename from crates/compiler-2/src/codegen/query.rs rename to crates/compiler/src/codegen/query.rs diff --git a/crates/compiler/src/compile/cargo_build.rs b/crates/compiler/src/compile/cargo_build.rs deleted file mode 100644 index d6a3814590..0000000000 --- a/crates/compiler/src/compile/cargo_build.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::{ - path::Path, - process::{Command, Output, Stdio}, -}; - -use crate::{ - compile::{CompileError, CompiledBinary}, - Verbosity, -}; - -pub fn build( - name: &str, - working_directory: &Path, - optimized: bool, - verbosity: Verbosity, -) -> Result { - rustfmt(working_directory); - - let mut cmd = Command::new("cargo"); - cmd.arg("build") - .arg("--manifest-path") - .arg(working_directory.join("Cargo.toml")) - .arg("--target=wasm32-unknown-unknown"); - - if optimized { - cmd.arg("--release"); - } - - verbosity.add_flags(&mut cmd); - - log::debug!("Executing {:?}", cmd); - - cmd.current_dir(working_directory); - - let status = cmd.status().map_err(CompileError::DidntStart)?; - - if !status.success() { - return Err(CompileError::BuildFailed(status)); - } - - log::debug!("Compiled successfully"); - - let config = if optimized { "release" } else { "debug" }; - - let wasm = working_directory - .join("target") - .join("wasm32-unknown-unknown") - .join(config) - .join(name.replace("-", "_")) - .with_extension("wasm"); - - std::fs::read(&wasm) - .map(CompiledBinary::from) - .map_err(|error| CompileError::UnableToReadBinary { path: wasm, error }) -} - -fn rustfmt(working_directory: &Path) { - let mut cmd = Command::new("cargo"); - cmd.arg("fmt") - .arg("--manifest-path") - .arg(working_directory.join("Cargo.toml")) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - log::debug!("Executing {:?}", cmd); - - let output = cmd.output(); - - match output { - Ok(Output { status, .. }) if status.success() => { - log::debug!("Formatted the generated code"); - }, - Ok(Output { status, stderr, .. }) => { - log::warn!( - "Rustfmt exited with status code: {}", - status.code().unwrap_or(1) - ); - let stderr = String::from_utf8_lossy(&stderr); - log::warn!("Stderr:\n{}", stderr); - }, - Err(e) => { - log::warn!( - "Unable to run \"cargo fmt\" on the generated project: {}", - e - ); - log::warn!("Is rustfmt installed?"); - }, - } -} diff --git a/crates/compiler/src/compile/components.rs b/crates/compiler/src/compile/components.rs deleted file mode 100644 index 04da6d3b33..0000000000 --- a/crates/compiler/src/compile/components.rs +++ /dev/null @@ -1,70 +0,0 @@ -use std::{ - error::Error, - fmt::{self, Display, Formatter}, - ops::Deref, - path::PathBuf, - process::ExitStatus, - sync::Arc, -}; - -use bytes::Bytes; - -#[derive(Debug, Clone, PartialEq)] -pub struct CompiledBinary(pub Bytes); - -impl From> for CompiledBinary { - fn from(bytes: Vec) -> Self { CompiledBinary(bytes.into()) } -} - -impl AsRef<[u8]> for CompiledBinary { - fn as_ref(&self) -> &[u8] { &self.0 } -} - -impl Deref for CompiledBinary { - type Target = Bytes; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -/// The result from compiling... Essentially a newtype'd `Result`. -#[derive(Debug)] -pub struct CompilationResult(pub Result>); - -#[derive(Debug)] -pub enum CompileError { - BuildFailed(ExitStatus), - DidntStart(std::io::Error), - UnableToReadBinary { - path: PathBuf, - error: std::io::Error, - }, -} - -impl Display for CompileError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - CompileError::BuildFailed(exit) => match exit.code() { - Some(code) => { - write!(f, "Compilation failed with exit code {}", code,) - }, - None => f.write_str("Compilation failed"), - }, - CompileError::DidntStart(_) => { - f.write_str("Unable to run the compiler. Is cargo installed?") - }, - CompileError::UnableToReadBinary { path, .. } => { - write!(f, "Unable to read \"{}\"", path.display()) - }, - } - } -} - -impl Error for CompileError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - CompileError::BuildFailed(_) => None, - CompileError::DidntStart(e) => Some(e), - CompileError::UnableToReadBinary { error, .. } => Some(error), - } - } -} diff --git a/crates/compiler/src/compile/mod.rs b/crates/compiler/src/compile/mod.rs deleted file mode 100644 index f3f0dfe793..0000000000 --- a/crates/compiler/src/compile/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -mod cargo_build; -mod components; -mod write_project_to_disk; - -use std::sync::Arc; - -pub use self::components::*; -use crate::{inputs::Inputs, BuildContext, codegen::Codegen}; - -#[salsa::query_group(CompileGroup)] -pub trait Compile: Inputs + Codegen { - #[salsa::dependencies] - fn build(&self) -> Result>; -} - -fn build(db: &dyn Compile) -> Result> { - let ctx = db.build_context(); - let files = db.files(); - - for file in &files { - write_project_to_disk::run(file, &ctx); - } - - let BuildContext { - name, - working_directory, - optimized, - verbosity, - .. - } = &*ctx; - - cargo_build::build(name, working_directory, *optimized, *verbosity) - .map_err(Arc::new) -} diff --git a/crates/compiler/src/compile/write_project_to_disk.rs b/crates/compiler/src/compile/write_project_to_disk.rs deleted file mode 100644 index b9965f79e7..0000000000 --- a/crates/compiler/src/compile/write_project_to_disk.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::{codegen::File, BuildContext}; - -pub(crate) fn run(File { path, data }: &File, ctx: &BuildContext) { - let full_path = ctx.working_directory.join(path); - - if let Some(parent) = full_path.parent() { - if let Err(e) = std::fs::create_dir_all(parent) { - log::error!( - "Unable to create the \"{}\" directory: {}", - parent.display(), - e - ); - return; - } - } - - log::debug!( - "Writing {} bytes to \"{}\"", - data.len(), - full_path.display() - ); - - if let Err(e) = std::fs::write(&full_path, data) { - log::error!("Unable to write to \"{}\": {}", full_path.display(), e); - } -} diff --git a/crates/compiler-2/src/config.rs b/crates/compiler/src/config.rs similarity index 100% rename from crates/compiler-2/src/config.rs rename to crates/compiler/src/config.rs diff --git a/crates/compiler/src/diagnostics.rs b/crates/compiler/src/diagnostics.rs deleted file mode 100644 index 806f8b6097..0000000000 --- a/crates/compiler/src/diagnostics.rs +++ /dev/null @@ -1,71 +0,0 @@ -use codespan_reporting::diagnostic::{Diagnostic, Severity}; - -type FileId = (); - -/// A collection of [`Diagnostic`]s. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct Diagnostics(Vec>); - -impl Diagnostics { - pub fn new() -> Self { Diagnostics(Vec::new()) } - - pub fn iter(&self) -> impl Iterator> + '_ { - self.0.iter() - } - - /// Get an iterator over all the [`Diagnostic`]s that are at least as severe - /// as a certain [`Severity`]. - pub fn iter_severity( - &self, - severity: Severity, - ) -> impl Iterator> + '_ { - self.iter().filter(move |diag| diag.severity >= severity) - } - - /// Are there any diagnostics which are at least as severe as a certain - /// [`Severity`] level? - pub fn has_severity(&self, severity: Severity) -> bool { - self.iter_severity(severity).next().is_some() - } - - /// Does this set of [`Diagnostics`] contain any [`Diagnostic`]s which are - /// at least as bad as an error? - pub fn has_errors(&self) -> bool { self.has_severity(Severity::Error) } - - /// Does this set of [`Diagnostics`] contain any [`Diagnostic`]s which are - /// at least as bad as a warning? - pub fn has_warnings(&self) -> bool { self.has_severity(Severity::Warning) } - - /// Add a new [`Diagnostic`] to the collection. - pub fn push(&mut self, diag: Diagnostic) { self.0.push(diag); } - - /// Is this collection of [`Diagnostic`]s empty? - pub fn is_empty(&self) -> bool { self.0.is_empty() } - - pub fn len(&self) -> usize { self.0.len() } - - /// Remove all [`Diagnostic`]s from this set of [`Diagnostics`]. - pub fn drain(&mut self) -> impl Iterator> + '_ { - self.0.drain(..) - } -} - -impl<'a> IntoIterator for &'a Diagnostics { - type IntoIter = <&'a Vec> as IntoIterator>::IntoIter; - type Item = &'a Diagnostic; - - fn into_iter(self) -> Self::IntoIter { self.0.iter() } -} - -impl IntoIterator for Diagnostics { - type IntoIter = > as IntoIterator>::IntoIter; - type Item = Diagnostic; - - fn into_iter(self) -> Self::IntoIter { self.0.into_iter() } -} - -impl Extend> for Diagnostics { - fn extend>>(&mut self, iter: T) { - self.0.extend(iter); - } -} diff --git a/crates/compiler-2/src/filesystem.rs b/crates/compiler/src/filesystem.rs similarity index 100% rename from crates/compiler-2/src/filesystem.rs rename to crates/compiler/src/filesystem.rs diff --git a/crates/compiler/src/hooks.rs b/crates/compiler/src/hooks.rs deleted file mode 100644 index 922a83bcf1..0000000000 --- a/crates/compiler/src/hooks.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! Callbacks that allow users to hook into the build process. - -use atomic_refcell::{AtomicRef, AtomicRefMut}; -use legion::{Resources, World}; - -use crate::{ - compile::CompilationResult, lowering::NameTable, parse::DocumentV1, - BuildContext, Diagnostics, FeatureFlags, -}; - -/// Callbacks that are fired at different points in the compilation process. -/// -/// Each hook is optional, with the default implementation returning -/// [`Continuation::Halt`] when [`Diagnostics::has_errors()`] indicates there -/// were errors. -pub trait Hooks { - /// Callback fired before the Runefile is parsed, giving the [`Hooks`] a - /// chance to do setup (e.g. by registering global [`Resources`] used in - /// later steps). - fn before_parse(&mut self, _ctx: &mut dyn Context) -> Continuation { - Continuation::Continue - } - - /// Callback fired after parsing the Runefile. - fn after_parse(&mut self, ctx: &mut dyn AfterParseContext) -> Continuation { - if ctx.diagnostics().has_errors() { - Continuation::Halt - } else { - Continuation::Continue - } - } - - /// Callback fired after lowering a [`crate::parse::Document`] to - /// [`crate::lowering`] types but before any type checking is applied. - fn after_lowering( - &mut self, - ctx: &mut dyn AfterLoweringContext, - ) -> Continuation { - if ctx.diagnostics().has_errors() { - Continuation::Halt - } else { - Continuation::Continue - } - } - - /// Callback fired after type checking and before codegen. - fn after_type_checking( - &mut self, - ctx: &mut dyn AfterTypeCheckingContext, - ) -> Continuation { - if ctx.diagnostics().has_errors() { - Continuation::Halt - } else { - Continuation::Continue - } - } - - /// Callback fired after generating the Rust project but immediately before - /// it is compiled to WebAssembly. - fn after_codegen( - &mut self, - ctx: &mut dyn AfterCodegenContext, - ) -> Continuation { - if ctx.diagnostics().has_errors() { - Continuation::Halt - } else { - Continuation::Continue - } - } - - fn after_compile( - &mut self, - ctx: &mut dyn AfterCompileContext, - ) -> Continuation { - if ctx.diagnostics().has_errors() { - Continuation::Halt - } else { - Continuation::Continue - } - } -} - -/// How to proceed after calling a [`Hooks`] method. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum Continuation { - /// Keep going. - Continue, - /// Stop. - Halt, -} - -/// Basic contextual information passed to all [`Hooks`]. -pub trait Context { - fn resources(&self) -> &Resources; - fn resources_mut(&mut self) -> &mut Resources; - fn world(&self) -> &World; - fn world_mut(&mut self) -> &mut World; - fn world_and_resources(&mut self) -> (&mut World, &mut Resources); - - fn build_context(&self) -> AtomicRef<'_, BuildContext> { - self.resources().get().unwrap() - } - - fn feature_flags(&self) -> AtomicRef<'_, FeatureFlags> { - self.resources().get().unwrap() - } -} - -/// Context passed to the [`Hooks::after_parse()`] method. -pub trait AfterParseContext: Context { - fn document(&self) -> AtomicRef<'_, DocumentV1> { - self.resources().get().unwrap() - } - - fn document_mut(&self) -> AtomicRefMut<'_, DocumentV1> { - self.resources().get_mut().unwrap() - } - - fn diagnostics(&self) -> AtomicRef<'_, Diagnostics> { - self.resources().get().unwrap() - } - - fn diagnostics_mut(&self) -> AtomicRefMut<'_, Diagnostics> { - self.resources().get_mut().unwrap() - } -} - -/// Context passed to the [`Hooks::after_lowering()`] method. -pub trait AfterLoweringContext: AfterParseContext { - fn names(&self) -> AtomicRef<'_, NameTable> { - self.resources().get().unwrap() - } -} - -/// Context passed to the [`Hooks::after_type_checking()`] method. -pub trait AfterTypeCheckingContext: AfterLoweringContext {} - -/// Context passed to the [`Hooks::after_codegen()`] method. -pub trait AfterCodegenContext: AfterTypeCheckingContext {} - -/// Context passed to the [`Hooks::after_compile()`] method. -pub trait AfterCompileContext: AfterCodegenContext { - fn take_compilation_result(&mut self) -> CompilationResult { - self.resources_mut().remove().unwrap() - } -} - -pub(crate) struct Ctx<'world, 'res> { - pub(crate) world: &'world mut World, - pub(crate) res: &'res mut Resources, -} - -impl<'world, 'res> Context for Ctx<'world, 'res> { - fn resources(&self) -> &Resources { self.res } - - fn resources_mut(&mut self) -> &mut Resources { self.res } - - fn world(&self) -> &World { self.world } - - fn world_mut(&mut self) -> &mut World { self.world } - - fn world_and_resources(&mut self) -> (&mut World, &mut Resources) { - (self.world, self.res) - } -} - -impl<'world, 'res> AfterParseContext for Ctx<'world, 'res> {} - -impl<'world, 'res> AfterLoweringContext for Ctx<'world, 'res> {} - -impl<'world, 'res> AfterTypeCheckingContext for Ctx<'world, 'res> {} - -impl<'world, 'res> AfterCodegenContext for Ctx<'world, 'res> {} - -impl<'world, 'res> AfterCompileContext for Ctx<'world, 'res> {} diff --git a/crates/compiler-2/src/im.rs b/crates/compiler/src/im.rs similarity index 100% rename from crates/compiler-2/src/im.rs rename to crates/compiler/src/im.rs diff --git a/crates/compiler/src/inputs.rs b/crates/compiler/src/inputs.rs deleted file mode 100644 index cf076f09e0..0000000000 --- a/crates/compiler/src/inputs.rs +++ /dev/null @@ -1,19 +0,0 @@ -use std::{path::Path, sync::Arc}; - -use bytes::Bytes; - -use crate::{BuildContext, FeatureFlags}; - -pub trait FileSystem { - fn read_file(&self, path: &Path) -> Result { - std::fs::read(path).map(Bytes::from) - } -} - -#[salsa::query_group(InputsGroup)] -pub trait Inputs: FileSystem { - #[salsa::input] - fn build_context(&self) -> Arc; - #[salsa::input] - fn feature_flags(&self) -> FeatureFlags; -} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index dfa909c42a..04a46577cf 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -1,50 +1,18 @@ -//! The Rune compiler. -//! -//! # Phases -//! -//! The compilation process is split into several phases which build on each -//! other, with user-injectable [`hooks`] called after each phase finishes. -//! -//! The phases are: -//! -//! 1. [`parse`] -//! 2. [`lowering`] -//! 3. [`type_check`] -//! 4. [`codegen`] -//! -//! # Stability -//! -//! This crate contains the internal types used by the Rune compiler so they can -//! be used externally. While this can give you a lot of flexibility and let you -//! extract a lot of information about a Rune, the compiler is a continually -//! evolving codebase. -//! -//! **This API should be considered unstable and subject to change.** +#![doc= include_str!("../README.md")] #[cfg(test)] #[macro_use] extern crate pretty_assertions; -#[cfg(test)] -#[macro_use] -mod macros; - -mod build_context; pub mod codegen; -pub mod compile; -mod diagnostics; -pub mod hooks; -pub mod lowering; +mod config; +mod filesystem; +pub mod im; pub mod parse; -mod phases; -pub mod serialize; -mod toolchain; pub mod type_check; -mod inputs; pub use crate::{ - build_context::{BuildContext, FeatureFlags, Verbosity}, - diagnostics::Diagnostics, - phases::{build, build_with_hooks, Phase}, - toolchain::rust_toolchain, + config::{BuildConfig, Environment, EnvironmentStorage}, + filesystem::{FileSystem, ReadError}, + im::Text, }; diff --git a/crates/compiler/src/lowering/components.rs b/crates/compiler/src/lowering/components.rs deleted file mode 100644 index 1d0406dfde..0000000000 --- a/crates/compiler/src/lowering/components.rs +++ /dev/null @@ -1,377 +0,0 @@ -//! The various types that make up Rune's *High-level Internal Representation*. - -use std::{ - borrow::{Borrow, Cow}, - fmt::{self, Display, Formatter}, - hash::Hash, - ops::Deref, - path::PathBuf, -}; - -use bytes::Bytes; -use hotg_rune_core::Shape; -use indexmap::IndexMap; -use legion::Entity; - -use crate::parse::{Path, ResourceType}; - -/// An output. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct Sink { - pub kind: SinkKind, - pub args: IndexMap, -} - -/// The kind of output. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case", tag = "type")] -pub enum SinkKind { - Serial, - Tensor, - Other(String), -} - -impl Display for SinkKind { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - SinkKind::Serial => write!(f, "serial"), - SinkKind::Tensor => write!(f, "tensor"), - SinkKind::Other(s) => write!(f, "{}", s), - } - } -} - -impl<'a> From<&'a str> for SinkKind { - fn from(s: &'a str) -> SinkKind { - match s { - "serial" | "SERIAL" => SinkKind::Serial, - "tensor" | "TENSOR" => SinkKind::Tensor, - _ => SinkKind::Other(s.to_string()), - } - } -} - -/// A ML model. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct Model { - pub model_file: ModelFile, - pub args: IndexMap, -} - -/// Where to load a model from. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum ModelFile { - /// Load the model from a file on disk. - FromDisk(PathBuf), - /// Load the model from a resource embedded/injected into the Rune. - Resource(Entity), -} - -/// Something which can generate data. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct Source { - pub kind: SourceKind, - pub parameters: IndexMap, -} - -/// Where should a [`Source`] pull its data from? -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case", tag = "type")] -pub enum SourceKind { - Random, - Accelerometer, - Sound, - Image, - Raw, - FloatImage, - Other(String), -} - -impl SourceKind { - pub const fn as_capability_index(&self) -> Option { - match self { - SourceKind::Random => Some(hotg_rune_core::capabilities::RAND), - SourceKind::Accelerometer => { - Some(hotg_rune_core::capabilities::ACCEL) - }, - SourceKind::Sound => Some(hotg_rune_core::capabilities::SOUND), - SourceKind::Image => Some(hotg_rune_core::capabilities::IMAGE), - SourceKind::Raw => Some(hotg_rune_core::capabilities::RAW), - SourceKind::FloatImage => { - Some(hotg_rune_core::capabilities::FLOAT_IMAGE) - }, - _ => None, - } - } - - pub fn as_capability_name(&self) -> Option<&'static str> { - self.as_capability_index() - .and_then(hotg_rune_core::capabilities::name) - } -} - -impl<'a> From<&'a str> for SourceKind { - fn from(s: &'a str) -> SourceKind { - match s { - "rand" | "RAND" => SourceKind::Random, - "accel" | "ACCEL" => SourceKind::Accelerometer, - "sound" | "SOUND" => SourceKind::Sound, - "image" | "IMAGE" => SourceKind::Image, - "raw" | "RAW" => SourceKind::Raw, - "float-image" | "FLOAT_IMAGE" => SourceKind::FloatImage, - _ => SourceKind::Other(s.to_string()), - } - } -} - -impl Display for SourceKind { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - SourceKind::Other(custom) => Display::fmt(custom, f), - _ => write!(f, "{:?}", self), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct ProcBlock { - pub path: Path, - pub parameters: IndexMap, -} - -impl ProcBlock { - /// The name of the Rust crate that implements this [`ProcBlock`]. - pub(crate) fn name(&self) -> &str { - let full_name = self.path.sub_path.as_ref().unwrap_or(&self.path.base); - let start_of_name = full_name.rfind('/').map(|ix| ix + 1).unwrap_or(0); - - &full_name[start_of_name..] - } -} - -// TODO: remove this -impl Hash for ProcBlock { - fn hash(&self, state: &mut H) { - self.path.hash(state); - self.parameters.iter().for_each(|pair| pair.hash(state)); - } -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct Resource { - /// Where to read the [`Resource`]'s default value from. - pub default_value: Option, - pub ty: ResourceType, -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum ResourceSource { - /// The value is specified in-line as a string. - Inline(String), - /// The value should be read from disk. - FromDisk(PathBuf), -} - -/// An identifier used to refer to an item in a Runefile. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub struct Name(String); - -impl> From for Name { - fn from(s: S) -> Self { Name(s.into()) } -} - -impl Deref for Name { - type Target = String; - - fn deref(&self) -> &String { &self.0 } -} - -impl Display for Name { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.write_str(&self.0) } -} - -impl Borrow for Name { - fn borrow(&self) -> &String { &self.0 } -} -impl Borrow for Name { - fn borrow(&self) -> &str { &self.0 } -} - -impl AsRef for Name -where - String: AsRef, -{ - fn as_ref(&self) -> &S { self.0.as_ref() } -} - -/// A lookup table mapping [`Name`] components back to their [`Entity`]. -#[derive( - Debug, Default, PartialEq, Clone, serde::Serialize, serde::Deserialize, -)] -pub struct NameTable(IndexMap); - -impl NameTable { - pub(crate) fn clear(&mut self) { self.0.clear(); } - - pub(crate) fn insert(&mut self, name: Name, ent: Entity) { - self.0.insert(name, ent); - } -} - -impl Deref for NameTable { - type Target = IndexMap; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -impl From> for NameTable { - fn from(m: IndexMap) -> Self { NameTable(m) } -} - -/// A tag component indicating this [`Entity`] is part of the Rune's pipeline. -#[derive( - Debug, - Default, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub struct PipelineNode; - -/// The [`Shape`] a tensor may take. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct Tensor(pub Shape<'static>); - -impl From> for Tensor { - fn from(s: Shape<'static>) -> Self { Tensor(s) } -} - -/// The list of [`Tensor`]s that may be the output from a [`PipelineNode`]. -#[derive( - Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize, -)] -pub struct Outputs { - pub tensors: Vec, -} - -/// The list of [`Tensor`]s that may be the inputs to a [`PipelineNode`]. -#[derive( - Debug, - Default, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub struct Inputs { - pub tensors: Vec, -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct ResourceData(pub Bytes); - -impl> From for ResourceData { - fn from(data: T) -> Self { ResourceData(data.into()) } -} - -impl AsRef<[u8]> for ResourceData { - fn as_ref(&self) -> &[u8] { &*self } -} - -impl Deref for ResourceData { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct ModelData(pub Bytes); - -impl> From for ModelData { - fn from(data: A) -> Self { ModelData(data.into()) } -} - -impl Deref for ModelData { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct Mimetype(Cow<'static, str>); - -impl Mimetype { - pub const ONNX: Mimetype = - Mimetype(Cow::Borrowed(hotg_rune_core::ONNX_MIMETYPE)); - pub const TENSORFLOW: Mimetype = - Mimetype(Cow::Borrowed(hotg_rune_core::TF_MIMETYPE)); - pub const TENSORFLOW_JS: Mimetype = - Mimetype(Cow::Borrowed(hotg_rune_core::TFJS_MIMETYPE)); - pub const TENSORFLOW_LITE: Mimetype = - Mimetype(Cow::Borrowed(hotg_rune_core::TFLITE_MIMETYPE)); -} - -impl Deref for Mimetype { - type Target = str; - - fn deref(&self) -> &Self::Target { &self.0 } -} - -impl Default for Mimetype { - fn default() -> Self { Mimetype::TENSORFLOW_LITE } -} - -impl From<&'static str> for Mimetype { - fn from(s: &'static str) -> Self { Mimetype(Cow::Borrowed(s)) } -} - -impl From for Mimetype { - fn from(s: String) -> Self { Mimetype(Cow::Owned(s)) } -} - -impl AsRef for Mimetype { - fn as_ref(&self) -> &str { &self.0 } -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum ResourceOrString { - String(String), - Resource(Entity), -} - -impl<'a> From<&'a str> for ResourceOrString { - fn from(s: &'a str) -> Self { ResourceOrString::String(s.into()) } -} - -impl From for ResourceOrString { - fn from(s: String) -> Self { ResourceOrString::String(s) } -} diff --git a/crates/compiler/src/lowering/load_model_data.rs b/crates/compiler/src/lowering/load_model_data.rs deleted file mode 100644 index f580eaa8d9..0000000000 --- a/crates/compiler/src/lowering/load_model_data.rs +++ /dev/null @@ -1,33 +0,0 @@ -use codespan::Span; -use legion::{systems::CommandBuffer, Entity}; - -use crate::{ - lowering::{Model, ModelData, ModelFile, Name}, - BuildContext, Diagnostics, -}; - -#[legion::system(for_each)] -pub(crate) fn run( - cmd: &mut CommandBuffer, - #[resource] diags: &mut Diagnostics, - #[resource] build_ctx: &BuildContext, - &entity: &Entity, - name: &Name, - model: &Model, - &span: &Span, -) { - match &model.model_file { - ModelFile::FromDisk(path) => { - match super::load_resource_data::load( - &build_ctx.current_directory, - path, - name, - span, - ) { - Ok(data) => cmd.add_component(entity, ModelData::from(data)), - Err(diag) => diags.push(diag), - } - }, - ModelFile::Resource(_) => {}, - } -} diff --git a/crates/compiler/src/lowering/load_resource_data.rs b/crates/compiler/src/lowering/load_resource_data.rs deleted file mode 100644 index 031306c417..0000000000 --- a/crates/compiler/src/lowering/load_resource_data.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{ - fs::{DirEntry, File}, - io::{Cursor, ErrorKind, Seek, Write}, - path::Path, -}; - -use codespan::Span; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use legion::{systems::CommandBuffer, Entity}; -use zip::{write::FileOptions, ZipWriter}; - -use crate::{ - lowering::{Name, Resource, ResourceData, ResourceSource}, - BuildContext, Diagnostics, -}; - -#[legion::system(for_each)] -pub(crate) fn run( - cmd: &mut CommandBuffer, - #[resource] diags: &mut Diagnostics, - #[resource] build_ctx: &BuildContext, - &entity: &Entity, - name: &Name, - resource: &Resource, - &span: &Span, -) { - let current_dir = &build_ctx.current_directory; - - match &resource.default_value { - Some(ResourceSource::FromDisk(path)) => { - match load(current_dir, path, name, span) { - Ok(data) => cmd.add_component(entity, ResourceData::from(data)), - Err(diag) => diags.push(diag), - } - }, - Some(ResourceSource::Inline(data)) => { - cmd.add_component(entity, ResourceData::from(data.clone())); - }, - None => {}, - } -} - -pub(crate) fn load( - current_dir: &Path, - filename: &Path, - name: &Name, - span: Span, -) -> Result, Diagnostic<()>> { - let full_path = current_dir.join(filename); - - let loaded = if full_path.is_dir() { - load_directory(&full_path) - } else { - std::fs::read(&full_path) - }; - - loaded.map_err(|e| read_failed_diagnostic(&full_path, name, e, span)) -} - -fn load_directory(full_path: &Path) -> Result, std::io::Error> { - let mut buffer = Cursor::new(Vec::new()); - - let mut archive = ZipWriter::new(&mut buffer); - - for entry in full_path.read_dir()? { - let entry = entry?; - append_entry(&mut archive, full_path, entry)?; - } - - archive.finish()?; - drop(archive); - - Ok(buffer.into_inner()) -} - -fn append_entry( - archive: &mut ZipWriter, - root: &Path, - entry: DirEntry, -) -> Result<(), std::io::Error> { - let path = entry.path(); - let relative_path = path - .strip_prefix(root) - .map_err(|e| std::io::Error::new(ErrorKind::Other, e))?; - let relative_path = relative_path.display().to_string(); - - let meta = entry.metadata()?; - let options = FileOptions::default(); - - if meta.is_dir() { - archive.add_directory(relative_path, options)?; - - for entry in path.read_dir()? { - let entry = entry?; - append_entry(archive, root, entry)?; - } - } else { - archive.start_file(relative_path, options)?; - let mut f = File::open(path)?; - std::io::copy(&mut f, archive)?; - } - - Ok(()) -} - -fn read_failed_diagnostic( - full_path: &Path, - name: &Name, - e: std::io::Error, - span: Span, -) -> Diagnostic<()> { - let msg = format!( - "Unable to read \"{}\" for \"{}\": {}", - full_path.display(), - name, - e - ); - - Diagnostic::error() - .with_message(msg) - .with_labels(vec![Label::primary((), span)]) -} diff --git a/crates/compiler/src/lowering/mod.rs b/crates/compiler/src/lowering/mod.rs deleted file mode 100644 index 4ced7ba71b..0000000000 --- a/crates/compiler/src/lowering/mod.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! The lowering phase. - -mod components; -mod load_model_data; -mod load_resource_data; -pub mod query; -mod register_names; -mod register_resources; -mod register_stages; -mod register_tensors; -mod update_nametable; - -use legion::Registry; - -pub use self::components::*; -use crate::{phases::Phase, serialize::RegistryExt}; - -pub fn phase() -> Phase { - Phase::with_setup(|res| { - res.insert(NameTable::default()); - }) - .and_then(register_names::run_system) - .and_then(update_nametable::run_system) - .and_then(register_resources::run_system) - .and_then(register_stages::run_system) - .and_then(register_tensors::run_system) - .and_then(load_resource_data::run_system) - .and_then(load_model_data::run_system) -} - -pub(crate) fn register_components(registry: &mut Registry) { - registry - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::() - .register_with_type_name::(); -} diff --git a/crates/compiler/src/lowering/query.rs b/crates/compiler/src/lowering/query.rs deleted file mode 100644 index a8527562b1..0000000000 --- a/crates/compiler/src/lowering/query.rs +++ /dev/null @@ -1,232 +0,0 @@ -use std::{num::NonZeroU32, sync::Arc}; - -use codespan::Span; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use im::HashMap; - -use crate::{ - inputs::FileSystem, - lowering::{Inputs, Outputs, Resource, ResourceSource}, - parse::{DocumentV1, Image, ResourceDeclaration, ResourceOrString}, - Diagnostics, -}; - -/// This is typically populated by something like [`apply_document()`]. -#[salsa::query_group(LoweringInputsGroup)] -pub trait Ast { - #[salsa::input] - fn image(&self) -> Image; - - #[salsa::input] - fn nodes(&self) -> HashMap, NodeId>; - #[salsa::input] - fn node(&self, id: NodeId) -> Node; - #[salsa::input] - fn node_inputs(&self, id: NodeId) -> Inputs; - #[salsa::input] - fn node_outputs(&self, id: NodeId) -> Outputs; - - #[salsa::input] - fn resources(&self) -> HashMap, ResourceId>; - #[salsa::input] - fn resource(&self, id: ResourceId) -> Resource; -} - -#[salsa::query_group(LoweringGroup)] -pub trait Lowering: Ast + FileSystem {} - -#[must_use = "Diagnostics should always be handled"] -pub fn apply_document(db: &mut dyn Ast, doc: &DocumentV1) -> Diagnostics { - let DocumentV1 { - version: _, - image, - pipeline, - resources, - } = doc; - - let mut diags = Diagnostics::new(); - - db.set_image(image.clone()); - - let mut ids = Counter::new(); - - set_nodes(pipeline, &mut ids, db); - set_resources(resources, &mut ids, db, &mut diags); - - diags -} - -fn set_resources( - resources: &indexmap::IndexMap, - ids: &mut Counter, - db: &mut dyn Ast, - diags: &mut Diagnostics, -) { - let mut res = HashMap::new(); - - for (name, decl) in resources { - let id = ids.resource_id(); - - let source = match (decl.inline.as_ref(), decl.path.as_ref()) { - (Some(inline), None) => { - Some(ResourceSource::Inline(inline.clone())) - }, - (None, Some(path)) => Some(ResourceSource::FromDisk(path.into())), - (None, None) => None, - (Some(_), Some(_)) => { - let diag = PathXorInlineResourceDiagnostic { - name: name.to_string(), - span: decl.span(), - }; - diags.push(diag.into_codespan_diagnostic()); - - continue; - }, - }; - - let resource = Resource { - default_value: source, - ty: decl.ty, - }; - res.insert(Arc::from(name.as_str()), id); - db.set_resource(id, resource); - } - - db.set_resources(res); -} - -#[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)] -#[error("A resource can't specify both \"path\" and \"inline\" values")] -#[diagnostic(code("E002"))] -pub struct PathXorInlineResourceDiagnostic { - pub name: String, - pub span: Span, -} - -impl PathXorInlineResourceDiagnostic { - fn into_codespan_diagnostic(&self) -> Diagnostic<()> { - let msg = format!( - "The resource \"{}\" can't specify both a \"path\" and \"inline\" \ - default value", - self.name - ); - - Diagnostic::error() - .with_message(msg) - .with_labels(vec![Label::primary((), self.span)]) - } -} - -fn set_nodes( - pipeline: &indexmap::IndexMap, - ids: &mut Counter, - db: &mut dyn Ast, -) { - let mut nodes = HashMap::new(); - for (name, stage) in pipeline { - let id = ids.node_id(); - let name: Arc = name.as_str().into(); - nodes.insert(name, id); - db.set_node(id, Node::from_stage(stage)); - } - db.set_nodes(nodes); -} - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct Node { - kind: NodeKind, - identifier: ResourceOrString, - arguments: HashMap, ResourceOrString>, -} - -impl Node { - fn from_stage(stage: &crate::parse::Stage) -> Self { - let (kind, identifier) = match stage { - crate::parse::Stage::Model(m) => (NodeKind::Model, m.model.clone()), - crate::parse::Stage::ProcBlock(_) => todo!(), - crate::parse::Stage::Capability(_) => todo!(), - crate::parse::Stage::Out(_) => todo!(), - }; - - let arguments = stage - .args() - .iter() - .map(|(k, v)| (Arc::from(k.as_str()), v.0.clone())) - .collect(); - - Node { - kind, - identifier, - arguments, - } - } -} - -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -pub enum NodeKind { - Capability, - Model, - ProcBlock, - Output, -} - -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -#[repr(transparent)] -pub struct NodeId(NonZeroU32); - -#[derive( - Debug, - Copy, - Clone, - PartialEq, - Eq, - Hash, - serde::Serialize, - serde::Deserialize, -)] -#[repr(transparent)] -pub struct ResourceId(NonZeroU32); - -#[derive( - Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub struct ArgumentId { - node: NodeId, - name: Arc, -} - -#[derive(Debug)] -struct Counter(u32); - -impl Counter { - fn new() -> Self { Counter(0) } - - fn next(&mut self) -> NonZeroU32 { - self.0 += 1; - NonZeroU32::new(self.0).expect("Unreachable") - } - - fn node_id(&mut self) -> NodeId { NodeId(self.next()) } - - fn resource_id(&mut self) -> ResourceId { ResourceId(self.next()) } -} diff --git a/crates/compiler/src/lowering/register_names.rs b/crates/compiler/src/lowering/register_names.rs deleted file mode 100644 index a87377e004..0000000000 --- a/crates/compiler/src/lowering/register_names.rs +++ /dev/null @@ -1,19 +0,0 @@ -use legion::systems::CommandBuffer; - -use crate::{ - lowering::{Name, PipelineNode}, - parse::DocumentV1, -}; - -/// Goes through and registers all the named items and their locations in the -/// Runefile. -#[legion::system] -pub(crate) fn run(cmd: &mut CommandBuffer, #[resource] doc: &DocumentV1) { - for (name, stage) in &doc.pipeline { - cmd.push((Name::from(name), stage.span(), PipelineNode)); - } - - for (name, decl) in &doc.resources { - cmd.push((Name::from(name), decl.span())); - } -} diff --git a/crates/compiler/src/lowering/register_resources.rs b/crates/compiler/src/lowering/register_resources.rs deleted file mode 100644 index 3b7944212c..0000000000 --- a/crates/compiler/src/lowering/register_resources.rs +++ /dev/null @@ -1,167 +0,0 @@ -use codespan::Span; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use legion::systems::CommandBuffer; - -use crate::{ - lowering::{NameTable, Resource, ResourceSource}, - parse::DocumentV1, - Diagnostics, -}; - -/// Register all the [`Resource`]s in a [`DocumentV1`]. -#[legion::system] -pub(crate) fn run( - cmd: &mut CommandBuffer, - #[resource] diags: &mut Diagnostics, - #[resource] doc: &mut DocumentV1, - #[resource] names: &NameTable, -) { - for (name, decl) in &doc.resources { - let ent = match names.get(name) { - Some(&e) => e, - None => { - // there was probably a duplicate name error in an earlier pass - // so this resource wasn't added to the name table. - continue; - }, - }; - - let source = match (decl.inline.as_ref(), decl.path.as_ref()) { - (Some(inline), None) => { - Some(ResourceSource::Inline(inline.clone())) - }, - (None, Some(path)) => Some(ResourceSource::FromDisk(path.into())), - (None, None) => None, - (Some(_), Some(_)) => { - diags.push(path_and_inline_defined_diagnostic( - name, - decl.span(), - )); - - continue; - }, - }; - - let resource = Resource { - default_value: source, - ty: decl.ty, - }; - - cmd.add_component(ent, resource); - } -} - -fn path_and_inline_defined_diagnostic( - name: &str, - span: Span, -) -> Diagnostic<()> { - let msg = format!( - "The resource \"{}\" can't specify both a \"path\" and \"inline\" \ - default value", - name - ); - - Diagnostic::error() - .with_message(msg) - .with_labels(vec![Label::primary((), span)]) -} - -#[cfg(never)] -mod tests { - use legion::{IntoQuery, Resources, World}; - - use super::*; - use crate::{ - lowering, - lowering::Name, - parse::{ResourceDeclaration, ResourceType}, - phases::Phase, - BuildContext, - }; - - fn doc() -> DocumentV1 { - DocumentV1 { - version: 1, - image: "img".parse().unwrap(), - pipeline: Default::default(), - resources: map! { - inline_string: ResourceDeclaration { - inline: Some("inline".to_string()), - path: None, - ty: ResourceType::String, - }, - path_bytes: ResourceDeclaration { - inline: None, - path: Some("data.bin".to_string()), - ty: ResourceType::Binary, - }, - no_defaults: ResourceDeclaration { - ty: ResourceType::Binary, - ..Default::default() - }, - error: ResourceDeclaration { - inline: Some("inline".to_string()), - path: Some("data.bin".to_string()), - ..Default::default() - } - }, - } - } - - #[test] - fn all_resources_are_registered() { - let mut world = World::default(); - let mut res = Resources::default(); - res.insert(BuildContext::from_doc(doc().into())); - res.insert(NameTable::default()); - let should_be = vec![ - ( - Name::from("inline_string"), - Resource { - default_value: Some(ResourceSource::Inline( - "inline".to_string(), - )), - ty: ResourceType::String, - }, - ), - ( - Name::from("path_bytes"), - Resource { - default_value: Some(ResourceSource::FromDisk( - "data.bin".into(), - )), - ty: ResourceType::Binary, - }, - ), - ( - Name::from("no_defaults"), - Resource { - default_value: None, - ty: ResourceType::Binary, - }, - ), - ]; - crate::parse::phase().run(&mut world, &mut res); - - Phase::new() - .and_then(lowering::register_names::run_system) - .and_then(lowering::update_nametable::run_system) - .and_then(run_system) - .run(&mut world, &mut res); - - let resources: Vec<_> = <(&Name, &Resource)>::query() - .iter(&world) - .map(|(n, r)| (n.clone(), r.clone())) - .collect(); - assert_eq!(resources, should_be); - - let diags = res.get::().unwrap(); - let diags: Vec<_> = diags.iter().collect(); - assert_eq!(diags.len(), 1); - assert_eq!( - diags[0].message, - "The resource \"error\" can't specify both a \"path\" and \ - \"inline\" default value", - ); - } -} diff --git a/crates/compiler/src/lowering/register_stages.rs b/crates/compiler/src/lowering/register_stages.rs deleted file mode 100644 index 1246a794a7..0000000000 --- a/crates/compiler/src/lowering/register_stages.rs +++ /dev/null @@ -1,489 +0,0 @@ -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use indexmap::IndexMap; -use legion::{systems::CommandBuffer, world::SubWorld, Entity, Query}; - -use crate::{ - lowering::{ - self, Mimetype, Model, ModelFile, NameTable, ProcBlock, Resource, - ResourceData, Sink, Source, - }, - parse::{ - self, CapabilityStage, DocumentV1, ModelStage, OutStage, - ProcBlockStage, ResourceName, ResourceType, - }, - Diagnostics, -}; - -/// Attach [`Model`], [`ProcBlock`], [`Sink`], and [`Source`] components to -/// each [`parse::Stage`] in the [`DocumentV1`]. -#[legion::system] -#[read_component(Resource)] -pub(crate) fn run( - cmd: &mut CommandBuffer, - world: &SubWorld, - #[resource] doc: &DocumentV1, - #[resource] names: &NameTable, - #[resource] diags: &mut Diagnostics, - resources: &mut Query<(&Resource, Option<&ResourceData>)>, -) { - for (name, stage) in &doc.pipeline { - let ent = match names.get(name) { - Some(&e) => e, - None => continue, - }; - - let args = match translate_args(stage.args(), names) { - Ok(a) => a, - Err(diag) => { - diags.push(diag); - continue; - }, - }; - - match stage { - parse::Stage::Model(ModelStage { model, .. }) => { - match register_model(names, name, model, &args, |e: Entity| { - resources.get(world, e).ok() - }) { - Ok((model, mimetype)) => { - cmd.add_component(ent, model); - cmd.add_component(ent, mimetype); - }, - Err(diag) => diags.push(diag), - } - }, - parse::Stage::ProcBlock(ProcBlockStage { proc_block, .. }) => { - if proc_block.version.is_none() { - let diag = warn_on_unversioned_proc_block_diagnostic( - name, proc_block, - ); - diags.push(diag); - } - - cmd.add_component( - ent, - ProcBlock { - path: proc_block.clone(), - parameters: args, - }, - ) - }, - parse::Stage::Capability(CapabilityStage { - capability, .. - }) => cmd.add_component( - ent, - Source { - kind: capability.as_str().into(), - parameters: args, - }, - ), - parse::Stage::Out(OutStage { out, .. }) => cmd.add_component( - ent, - Sink { - kind: out.as_str().into(), - args, - }, - ), - } - } -} - -fn warn_on_unversioned_proc_block_diagnostic( - name: &str, - proc_block: &parse::Path, -) -> Diagnostic<()> { - let msg = format!( - "The \"{}\" proc block used by \"{}\" should have a version specifier", - proc_block, name - ); - let versioned = parse::Path { - version: Some(env!("CARGO_PKG_VERSION").to_string()), - ..proc_block.clone() - }; - - Diagnostic::warning() - .with_message(msg) - .with_notes(vec![format!( - "hint: change it to something like \"{}\"", - versioned - )]) -} - -fn translate_args( - args: &IndexMap, - names: &NameTable, -) -> Result, Diagnostic<()>> { - let mut translated = IndexMap::new(); - - for (name, value) in args { - let value = match &value.0 { - parse::ResourceOrString::Resource(r) => match names - .get(r.as_str()) - .copied() - { - Some(entity) => lowering::ResourceOrString::Resource(entity), - None => return Err(not_a_resource_diagnostic(r)), - }, - parse::ResourceOrString::String(s) => { - lowering::ResourceOrString::String(s.clone()) - }, - }; - - translated.insert(name.clone(), value); - } - - Ok(translated) -} - -fn register_model<'a>( - names: &NameTable, - node_name: &str, - model: &parse::ResourceOrString, - args: &IndexMap, - mut get_resource: impl FnMut(Entity) -> Option<(&'a Resource, Option<&'a ResourceData>)> - + 'a, -) -> Result<(Model, Mimetype), Diagnostic<()>> { - let (mimetype, args) = model_format_and_args(node_name, args, |e| { - get_resource(e).and_then(|r| r.1).cloned() - })?; - - let model_file = match model { - parse::ResourceOrString::Resource(resource_name) => { - resource_model(resource_name, names, |e| { - get_resource(e).map(|r| r.0) - })? - }, - parse::ResourceOrString::String(s) => ModelFile::FromDisk(s.into()), - }; - - Ok((Model { model_file, args }, mimetype)) -} - -fn model_format_and_args( - node_name: &str, - args: &IndexMap, - get_resource_data: impl FnOnce(Entity) -> Option, -) -> Result< - (Mimetype, IndexMap), - Diagnostic<()>, -> { - let mut args = args.clone(); - - let mimetype = match args.remove("format") { - Some(lowering::ResourceOrString::String(format)) => { - mimetype_for_known_format(&format)? - }, - Some(lowering::ResourceOrString::Resource(entity)) => { - match get_resource_data(entity) { - Some(data) => match std::str::from_utf8(&data) { - Ok(format) => mimetype_for_known_format(format)?, - Err(e) => { - return Err(invalid_mimetype_diagnostic(node_name, e)) - }, - }, - None => { - todo!("Handle unknown resource in the format") - }, - } - }, - None => Mimetype::default(), - }; - - Ok((mimetype, args)) -} - -fn invalid_mimetype_diagnostic( - node_name: &str, - e: std::str::Utf8Error, -) -> Diagnostic<()> { - let msg = format!("Invalid format for \"{}\": {}", node_name, e); - - Diagnostic::error().with_message(msg) -} - -fn mimetype_for_known_format(format: &str) -> Result> { - let known_formats = [ - ("onnx", hotg_rune_core::ONNX_MIMETYPE), - ("tensorflow", hotg_rune_core::TF_MIMETYPE), - ("tensorflow-js", hotg_rune_core::TFJS_MIMETYPE), - ("tensorflow-lite", hotg_rune_core::TFLITE_MIMETYPE), - ]; - - known_formats - .iter() - .find(|(name, _)| *name == format) - .map(|(_, mt)| Mimetype::from(*mt)) - .ok_or_else(|| { - unknown_format_diagnostic( - &format, - known_formats.iter().copied().map(|(f, _)| f), - ) - }) -} - -fn unknown_format_diagnostic( - format: &str, - expected: impl Iterator, -) -> Diagnostic<()> { - // TODO: use span information to tell the user where the error came from - - let msg = format!( - "Expected the format to be one of {}, but found {:?}", - join(expected, ", "), - format - ); - Diagnostic::error().with_message(msg) -} - -fn join<'a>(items: impl Iterator, separator: &str) -> String { - let mut buffer = String::new(); - - for (i, item) in items.enumerate() { - if i > 0 { - buffer.push_str(separator); - } - - buffer.push_str(item); - } - - buffer -} - -fn resource_model<'a>( - resource_name: &parse::ResourceName, - names: &NameTable, - get_resource: impl FnOnce(Entity) -> Option<&'a Resource> + 'a, -) -> Result> { - let ent = match names.get(resource_name.as_str()) { - Some(&e) => e, - None => return Err(unknown_resource_diagnostic(resource_name)), - }; - - let res = match get_resource(ent) { - Some(r) => r, - None => return Err(not_a_resource_diagnostic(resource_name)), - }; - - if res.ty != ResourceType::Binary { - return Err(model_resource_should_be_binary_diagnostic(resource_name)); - } - - Ok(ModelFile::Resource(ent)) -} - -fn model_resource_should_be_binary_diagnostic( - resource_name: &ResourceName, -) -> Diagnostic<()> { - Diagnostic::error() - .with_message(format!( - "\"{}\" should be a binary resource", - resource_name - )) - .with_labels(vec![Label::primary((), resource_name.span())]) -} - -fn not_a_resource_diagnostic(resource_name: &ResourceName) -> Diagnostic<()> { - Diagnostic::error() - .with_message(format!("\"{}\" is not a resource", resource_name)) - .with_labels(vec![Label::primary((), resource_name.span())]) -} - -fn unknown_resource_diagnostic(resource_name: &ResourceName) -> Diagnostic<()> { - Diagnostic::error() - .with_message(format!("No definition for \"{}\"", resource_name)) - .with_labels(vec![Label::primary((), resource_name.span())]) -} - -#[cfg(never)] -mod tests { - use indexmap::IndexMap; - use legion::{IntoQuery, Resources, World}; - - use super::*; - use crate::{ - lowering::{self, Name, SinkKind, SourceKind}, - parse::{ResourceDeclaration, ResourceType, Stage}, - phases::Phase, - BuildContext, - }; - - fn doc() -> DocumentV1 { - DocumentV1 { - version: 1, - image: "img".parse().unwrap(), - pipeline: map! { - cap: Stage::Capability(CapabilityStage { - capability: "SOUND".to_string(), - args: map! { - hz: "128".into(), - }, - outputs: Vec::new(), - }), - transform: Stage::ProcBlock(ProcBlockStage { - proc_block: "my-proc-block".parse().unwrap(), - args: map! { - some_arg: "asdf".into(), - }, - inputs: Vec::new(), - outputs: Vec::new(), - }), - model_from_disk: Stage::Model(ModelStage { - model: parse::ResourceOrString::String("model.tflite".into()), - inputs: Vec::new(), - outputs: Vec::new(), - args: IndexMap::new(), - }), - model_from_resource: Stage::Model(ModelStage { - model: parse::ResourceOrString::Resource("$MODEL_FILE".parse().unwrap()), - inputs: Vec::new(), - outputs: Vec::new(), - args: IndexMap::new(), - }), - model_with_not_a_resource: Stage::Model(ModelStage { - model: parse::ResourceOrString::Resource("$cap".parse().unwrap()), - inputs: Vec::new(), - outputs: Vec::new(), - args: IndexMap::new(), - }), - model_with_missing_resource: Stage::Model(ModelStage { - model: parse::ResourceOrString::Resource("$NON_EXISTENT".parse().unwrap()), - inputs: Vec::new(), - outputs: Vec::new(), - args: IndexMap::new(), - }), - model_with_string_resource: Stage::Model(ModelStage { - model: parse::ResourceOrString::Resource("$STRING_RESOURCE".parse().unwrap()), - inputs: Vec::new(), - outputs: Vec::new(), - args: IndexMap::new(), - }), - serial: Stage::Out(OutStage { - out: "SERIAL".to_string(), - args: Default::default(), - inputs: Vec::new(), - }), - }, - resources: map! { - MODEL_FILE: ResourceDeclaration { - inline: None, - path: Some("model.tflite".to_string()), - ty: ResourceType::Binary, - }, - STRING_RESOURCE: ResourceDeclaration { - inline: Some("res".to_string()), - path: None, - ty: ResourceType::String, - }, - }, - } - } - - #[test] - fn register_all_stages() { - let mut world = World::default(); - let mut res = Resources::default(); - res.insert(BuildContext::from_doc(doc().into())); - res.insert(NameTable::default()); - crate::parse::phase().run(&mut world, &mut res); - - Phase::new() - .and_then(lowering::register_names::run_system) - .and_then(lowering::update_nametable::run_system) - .and_then(lowering::register_resources::run_system) - .and_then(run_system) - .run(&mut world, &mut res); - - let diags = res.get::().unwrap(); - let diags: Vec<_> = diags.iter().collect(); - assert_eq!(diags.len(), 4); - assert_eq!( - diags[0], - &Diagnostic::warning() - .with_message( - "The \"my-proc-block\" proc block used by \"transform\" \ - should have a version specifier" - ) - .with_notes(vec![format!( - "hint: change it to something like \"my-proc-block@{}\"", - env!("CARGO_PKG_VERSION").to_string() - ) - .to_string()]) - ); - assert_eq!(diags[1].message, "\"$cap\" is not a resource"); - assert_eq!(diags[2].message, "No definition for \"$NON_EXISTENT\""); - assert_eq!( - diags[3].message, - "\"$STRING_RESOURCE\" should be a binary resource" - ); - - let proc_blocks_should_be = vec![( - Name::from("transform"), - ProcBlock { - path: "my-proc-block".parse().unwrap(), - parameters: map! { - some_arg: "asdf".into(), - }, - }, - )]; - let got: Vec<_> = <(&Name, &ProcBlock)>::query() - .iter(&world) - .map(|(n, p)| (n.clone(), p.clone())) - .collect(); - assert_eq!(got, proc_blocks_should_be); - - let models_should_be = vec![ - ( - Name::from("model_from_disk"), - Model { - model_file: ModelFile::FromDisk("model.tflite".into()), - args: IndexMap::new(), - }, - ), - ( - Name::from("model_from_resource"), - Model { - model_file: ModelFile::Resource( - *res.get::() - .unwrap() - .get("MODEL_FILE") - .unwrap(), - ), - args: IndexMap::new(), - }, - ), - ]; - let got: Vec<_> = <(&Name, &Model)>::query() - .iter(&world) - .map(|(n, m)| (n.clone(), m.clone())) - .collect(); - assert_eq!(got, models_should_be); - - let sources_should_be = vec![( - Name::from("cap"), - Source { - kind: SourceKind::Sound, - parameters: map! { - hz: "128".into(), - }, - }, - )]; - let got: Vec<_> = <(&Name, &Source)>::query() - .iter(&world) - .map(|(n, s)| (n.clone(), s.clone())) - .collect(); - assert_eq!(got, sources_should_be); - - let sinks_should_be = vec![( - Name::from("serial"), - Sink { - kind: SinkKind::Serial, - args: map! {}, - }, - )]; - let got: Vec<_> = <(&Name, &Sink)>::query() - .iter(&world) - .map(|(n, s)| (n.clone(), s.clone())) - .collect(); - assert_eq!(got, sinks_should_be); - } -} diff --git a/crates/compiler/src/lowering/register_tensors.rs b/crates/compiler/src/lowering/register_tensors.rs deleted file mode 100644 index 2d560f1452..0000000000 --- a/crates/compiler/src/lowering/register_tensors.rs +++ /dev/null @@ -1,320 +0,0 @@ -use std::collections::HashMap; - -use codespan_reporting::diagnostic::Diagnostic; -use hotg_rune_core::{ElementType, Shape}; -use legion::{systems::CommandBuffer, Entity}; - -use crate::{ - lowering::{Inputs, NameTable, Outputs, Tensor}, - parse::{self, DocumentV1}, - Diagnostics, -}; - -/// Register all [`Tensor`]s and associate them as node [`Inputs`] or -/// [`Outputs`]. -#[legion::system] -pub(crate) fn run( - cmd: &mut CommandBuffer, - #[resource] names: &NameTable, - #[resource] doc: &DocumentV1, - #[resource] diags: &mut Diagnostics, -) { - let node_outputs = register_node_outputs(cmd, names, doc, diags); - let node_inputs = - register_node_inputs(doc, names, &node_outputs, cmd, diags); - - for (&node, outputs) in &node_outputs { - for &tensor in &outputs.tensors { - let inputs = Inputs { - tensors: vec![node], - }; - - let outputs: Vec<_> = node_inputs - .iter() - .filter_map(|(&ent, inputs)| { - if inputs.tensors.contains(&tensor) { - Some(ent) - } else { - None - } - }) - .collect(); - - cmd.add_component(tensor, inputs); - cmd.add_component(tensor, Outputs { tensors: outputs }); - } - } -} - -fn register_node_inputs( - doc: &DocumentV1, - names: &NameTable, - output_tensors_by_node: &HashMap, - cmd: &mut CommandBuffer, - diags: &mut Diagnostics, -) -> HashMap { - let mut outputs = HashMap::new(); - - for (name, stage) in &doc.pipeline { - let ent = match names.get(name) { - Some(&e) => e, - None => continue, - }; - - match register_stage_inputs( - name, - stage.inputs(), - names, - output_tensors_by_node, - ) { - Ok(inputs) if inputs.tensors.is_empty() => {}, - Ok(inputs) => { - cmd.add_component(ent, inputs.clone()); - outputs.insert(ent, inputs); - }, - Err(diag) => diags.push(diag), - } - } - - outputs -} - -fn register_stage_inputs( - parent_name: &str, - inputs: &[parse::Input], - names: &NameTable, - output_tensors_by_node: &HashMap, -) -> Result> { - let mut tensors = Vec::new(); - - for input in inputs { - let tensor = get_input_tensor( - parent_name, - input, - names, - output_tensors_by_node, - )?; - tensors.push(tensor); - } - - Ok(Inputs { tensors }) -} - -fn get_input_tensor( - parent_name: &str, - input: &parse::Input, - names: &NameTable, - output_tensors_by_node: &HashMap, -) -> Result> { - // Find the node this "Input" refers to - let input_node = names - .get(&input.name) - .copied() - .ok_or_else(|| unknown_input_name_diagnostic(parent_name, input))?; - - // Then get its set of Outputs - let output_tensors = output_tensors_by_node - .get(&input_node) - .ok_or_else(|| node_has_no_outputs_diagnostic(parent_name, input))?; - - // Finally, get the Entity for the index'th item - let tensor = output_tensors - .tensors - .get(input.index.unwrap_or(0)) - .copied() - .ok_or_else(|| no_such_output_diagnostic(input))?; - - Ok(tensor) -} - -fn no_such_output_diagnostic(input: &parse::Input) -> Diagnostic<()> { - Diagnostic::error().with_message(format!( - "The \"{}\" node has no {}'th output", - input.name, - input.index.unwrap_or(0) - )) -} - -fn node_has_no_outputs_diagnostic( - parent_name: &str, - input: &parse::Input, -) -> Diagnostic<()> { - Diagnostic::error().with_message(format!( - "The \"{}\" in {}'s \"{}\" input has no inputs", - input.name, parent_name, input, - )) -} - -fn unknown_input_name_diagnostic( - parent_name: &str, - input: &parse::Input, -) -> Diagnostic<()> { - Diagnostic::error().with_message(format!( - "Unable to find \"{}\" to use as an input for \"{}\"", - input, parent_name, - )) -} - -fn register_node_outputs( - cmd: &mut CommandBuffer, - names: &NameTable, - doc: &DocumentV1, - diags: &mut Diagnostics, -) -> HashMap { - let mut node_to_output_tensors = HashMap::new(); - - for (name, stage) in &doc.pipeline { - let ent = match names.get(name) { - Some(&e) => e, - None => continue, - }; - - match allocate_output_tensors(cmd, stage.output_types()) { - Ok(outputs) if outputs.tensors.is_empty() => {}, - Ok(outputs) => { - node_to_output_tensors.insert(ent, outputs.clone()); - cmd.add_component(ent, outputs); - }, - Err(diag) => diags.push(diag), - } - } - - node_to_output_tensors -} - -/// Allocate a new [`Tensor`] entity for each output that a node may have. -fn allocate_output_tensors( - cmd: &mut CommandBuffer, - output_types: &[parse::Type], -) -> Result> { - let mut outputs = Vec::new(); - - for ty in output_types { - let tensor = shape(ty)?; - outputs.push(cmd.push((tensor,))); - } - - Ok(Outputs { tensors: outputs }) -} - -fn shape(ty: &parse::Type) -> Result> { - let element_type: ElementType = ty - .name - .to_lowercase() - .parse() - .map_err(|_| unknown_element_type_diagnostic(&ty.name))?; - - Ok(Tensor::from(Shape::new( - element_type, - ty.dimensions.clone(), - ))) -} - -fn unknown_element_type_diagnostic(name: &str) -> Diagnostic<()> { - Diagnostic::error() - .with_message(format!("Unknown element type, \"{}\"", name)) -} - -#[cfg(never)] -mod tests { - use std::str::FromStr; - - use legion::{IntoQuery, Resources, World}; - - use super::*; - use crate::{ - lowering::{self, PipelineNode}, - parse::{CapabilityStage, OutStage, ProcBlockStage}, - phases::Phase, - BuildContext, - }; - - fn doc() -> DocumentV1 { - DocumentV1 { - version: 1, - image: "image".parse().unwrap(), - pipeline: map! { - rand: parse::Stage::Capability(CapabilityStage { - capability: "RAND".to_string(), - outputs: vec![ - ty!(f32[128]), - ], - args: map! {}, - }), - transform: parse::Stage::ProcBlock(ProcBlockStage { - proc_block: "proc-block@1.0".parse().unwrap(), - inputs: vec![ - "rand".parse().unwrap(), - ], - outputs: vec![ - ty!(u8[1]), - ty!(u8[2]), - ], - args: map! {}, - }), - output: parse::Stage::Out(OutStage { - out: "SERIAL".to_string(), - inputs: vec![ - "transform.1".parse().unwrap(), - "transform.0".parse().unwrap(), - ], - args: map! {}, - }) - }, - resources: map! {}, - } - } - - #[test] - fn construct_pipeline() { - let mut world = World::default(); - let mut res = Resources::default(); - res.insert(BuildContext::from_doc(doc().into())); - res.insert(NameTable::default()); - crate::parse::phase().run(&mut world, &mut res); - - Phase::new() - .and_then(lowering::register_names::run_system) - .and_then(lowering::update_nametable::run_system) - .and_then(lowering::register_stages::run_system) - .and_then(run_system) - .run(&mut world, &mut res); - - let diags = res.get::().unwrap(); - assert!(diags.is_empty()); - - let names = res.get::().unwrap(); - let connections = vec![ - (("rand", 0), ("transform", 0), "f32[128]"), - (("transform", 0), ("output", 1), "u8[1]"), - (("transform", 1), ("output", 0), "u8[2]"), - ]; - let mut inputs = - <&Inputs>::query().filter(legion::component::()); - let mut outputs = - <&Outputs>::query().filter(legion::component::()); - let mut tensors = <&Tensor>::query(); - - for ((prev_name, prev_ix), (next_name, next_ix), ty) in connections { - let ty_should_be = Tensor::from(Shape::from_str(ty).unwrap()); - - let prev = names[prev_name]; - let outputs = outputs.get(&world, prev).unwrap(); - let output_tensor = outputs.tensors[prev_ix]; - assert_eq!( - tensors.get(&world, output_tensor).unwrap(), - &ty_should_be - ); - - let next = names[next_name]; - let inputs = inputs.get(&world, next).unwrap(); - let input_tensor = inputs.tensors[next_ix]; - assert_eq!( - tensors.get(&world, input_tensor).unwrap(), - &ty_should_be - ); - - assert_eq!(input_tensor, output_tensor); - } - } -} diff --git a/crates/compiler/src/lowering/update_nametable.rs b/crates/compiler/src/lowering/update_nametable.rs deleted file mode 100644 index 386883637d..0000000000 --- a/crates/compiler/src/lowering/update_nametable.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::collections::HashMap; - -use codespan::Span; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use legion::{world::SubWorld, Entity, Query}; - -use crate::{ - lowering::{Name, NameTable}, - Diagnostics, -}; - -/// Update the [`NameTable`] resource so we can track all the named items in a -/// Runefile. -#[legion::system] -pub(crate) fn run( - world: &mut SubWorld, - #[resource] diags: &mut Diagnostics, - #[resource] names: &mut NameTable, - named_items: &mut Query<(Entity, &Name, &Span)>, -) { - names.clear(); - - let mut lookup_table: HashMap<&Name, Vec<_>> = HashMap::new(); - - named_items.for_each(world, |(e, n, s)| { - let items = lookup_table.entry(n).or_default(); - items.push((e, s)); - // Note: Keep them sorted by location in the source file so the - // "first definition was here" message points at the item closest to the - // top. - items.sort_by_key(|(_, &s)| s); - }); - - for (name, items) in lookup_table { - match items.as_slice() { - [] => unreachable!(), - [(&ent, _)] => { - // The happy path - the file had just one item with this name. - names.insert(name.clone(), ent); - }, - [(&ent, &first_definition), others @ ..] => { - // emit an error message and only remember the first - let diag = duplicate_name_diagnostic( - name, - first_definition, - others.iter().map(|(_, &s)| s), - ); - diags.push(diag); - - names.insert(name.clone(), ent); - }, - } - } -} - -fn duplicate_name_diagnostic( - name: &Name, - first_definition: Span, - duplicates: impl Iterator, -) -> Diagnostic<()> { - let primary = Label::primary((), first_definition) - .with_message("The first definition is here"); - - let mut labels = vec![primary]; - - for duplicate in duplicates { - labels.push( - Label::secondary((), duplicate).with_message("Redefined here"), - ); - } - - Diagnostic::error() - .with_message(format!( - "The name \"{}\" is defined multiple times", - name - )) - .with_labels(labels) -} diff --git a/crates/compiler/src/macros.rs b/crates/compiler/src/macros.rs deleted file mode 100644 index 3cfbf97ff8..0000000000 --- a/crates/compiler/src/macros.rs +++ /dev/null @@ -1,29 +0,0 @@ -macro_rules! map { - // map-like - ($($k:ident : $v:expr),* $(,)?) => { - std::iter::Iterator::collect(IntoIterator::into_iter([ - $( - (String::from(stringify!($k)), $v) - ),* - ])) - }; - // set-like - ($($v:expr),* $(,)?) => { - std::iter::Iterator::collect(std::array::IntoIter::new([$($v,)*])) - }; -} - -macro_rules! ty { - ($type:ident [$($dim:expr),*]) => { - crate::parse::Type { - name: String::from(stringify!($type)), - dimensions: vec![ $($dim),*], - } - }; - ($type:ident) => { - crate::parse::Type { - name: String::from(stringify!($type)), - dimensions: vec![], - } - } - } diff --git a/crates/compiler/src/parse/mod.rs b/crates/compiler/src/parse/mod.rs index 12d6ec247d..24191fc66e 100644 --- a/crates/compiler/src/parse/mod.rs +++ b/crates/compiler/src/parse/mod.rs @@ -1,32 +1,82 @@ +//! The YAML frontend for the Rune compiler. +//! +//! You are probably here for either the [`Frontend`] trait or the [`Document`] +//! type. + +mod query; mod yaml; use std::sync::Arc; -use codespan_reporting::diagnostic::{Diagnostic, Label}; +use crate::Text; + +pub use self::{ + query::{Frontend, FrontendStorage}, + yaml::*, +}; -pub use self::yaml::*; +#[derive(Debug, Clone, thiserror::Error)] +#[error("Unable to parse the Runefile")] +pub struct ParseFailed { + #[from] + pub error: Arc, +} -#[tracing::instrument(skip(src), err)] -pub fn parse(src: &str) -> Result { - Document::parse(src) - .map_err(|e| ParseFailedDiagnostic { inner: Arc::new(e) }) +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, +)] +#[serde(rename_all = "kebab-case")] +pub enum ItemType { + Input, + Model, + ProcBlock, + Output, + Resource, } -#[derive(Debug, Clone, thiserror::Error, miette::Diagnostic)] -#[error("Unable to parse the Runefile: {}", inner)] -#[diagnostic(code("P001"))] -pub struct ParseFailedDiagnostic { - #[source] - inner: Arc, +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + thiserror::Error, + serde::Serialize, + serde::Deserialize, +)] +#[error("There is no model called \"{}\"", name)] +#[serde(rename_all = "kebab-case")] +pub struct NotFound { + pub item_type: ItemType, + pub name: Text, } -impl ParseFailedDiagnostic { - pub fn as_codespan_diagnostic(&self) -> Diagnostic<()> { - let mut diag = Diagnostic::error().with_message(self.to_string()); - if let Some(location) = self.inner.location() { - let ix = location.index(); - diag = diag.with_labels(vec![Label::primary((), ix..ix)]); - } - diag - } +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Hash, + thiserror::Error, + serde::Serialize, + serde::Deserialize, +)] +#[error( + "Expected \"{}\" to be a {:?}, but it is actually a {:?}", + name, + expected, + actual +)] +#[serde(rename_all = "kebab-case")] +pub struct WrongItemType { + pub expected: ItemType, + pub actual: ItemType, + pub name: Text, } diff --git a/crates/compiler-2/src/parse/query.rs b/crates/compiler/src/parse/query.rs similarity index 99% rename from crates/compiler-2/src/parse/query.rs rename to crates/compiler/src/parse/query.rs index 4fde2df4a2..2519bba86f 100644 --- a/crates/compiler-2/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -20,7 +20,7 @@ use crate::{ /// [`Frontend::parse()`]. /// /// ```rust -/// use hotg_rune_compiler_2::{ +/// use hotg_rune_compiler::{ /// parse::{Frontend, FrontendStorage}, /// EnvironmentStorage, FileSystem, ReadError, parse::Path, im::Vector, /// }; diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index 7d09b03bb6..fb902e54c7 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -7,7 +7,6 @@ use std::{ str::FromStr, }; -use codespan::Span; use indexmap::IndexMap; use once_cell::sync::Lazy; use regex::Regex; @@ -22,6 +21,7 @@ use serde::{ de::{Deserialize, Deserializer, Error as _}, ser::{Serialize, Serializer}, }; +use uriparse::{URIError, URI}; static RESOURCE_NAME_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^\$[_a-zA-Z][_a-zA-Z0-9]*$").unwrap()); @@ -42,7 +42,9 @@ impl Document { } impl From for Document { - fn from(v1: DocumentV1) -> Self { Document::V1(v1) } + fn from(v1: DocumentV1) -> Self { + Document::V1(v1) + } } mod document_serde { @@ -59,7 +61,9 @@ mod document_serde { } impl Repr { - fn new(version: usize, inner: T) -> Self { Repr { version, inner } } + fn new(version: usize, inner: T) -> Self { + Repr { version, inner } + } } impl Serialize for Document { @@ -104,7 +108,9 @@ mod document_serde { macro_rules! impl_json_schema_via_regex { ($ty:ty, $pattern:expr, $docs:literal) => { impl JsonSchema for $ty { - fn schema_name() -> String { String::from(stringify!($ty)) } + fn schema_name() -> String { + String::from(stringify!($ty)) + } fn json_schema(_: &mut SchemaGenerator) -> Schema { let mut schema = SchemaObject { @@ -130,6 +136,7 @@ macro_rules! impl_json_schema_via_regex { Debug, Clone, PartialEq, + Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, @@ -167,143 +174,17 @@ impl Document { impl FromStr for Document { type Err = serde_yaml::Error; - fn from_str(s: &str) -> Result { Document::parse(s) } -} - -/// A specification for finding a dependency. -/// -/// The full syntax is `base@version#sub_path` where -/// -/// - `base` is a URL or the name of a repository on GitHub (e.g. `hotg-ai/rune` -/// or `https://github.com/hotg-ai/rune`) -/// - `version` is an optional field specifying the version (e.g. as a git tag) -/// - `sub_path` is an optional field which is useful when pointing to -/// repositories with multiple relevant items because it lets you specify -/// which directory the specified item is in. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct Path { - pub base: String, - pub sub_path: Option, - pub version: Option, -} - -impl_json_schema_via_regex!( - Path, - PATH_PATTERN, - r#" -A specification for finding a dependency. - -The full syntax is `base@version#sub_path` where - -- `base` is a URL or the name of a repository on GitHub (e.g. `hotg-ai/rune` - or `https://github.com/hotg-ai/rune`) -- `version` is an optional field specifying the version (e.g. as a git tag) -- `sub_path` is an optional field which is useful when pointing to - repositories with multiple relevant items because it lets you specify - which directory the specified item is in. -"# -); - -impl Path { - pub fn new( - base: impl Into, - sub_path: impl Into>, - version: impl Into>, - ) -> Self { - Path { - base: base.into(), - sub_path: sub_path.into(), - version: version.into(), - } - } -} - -impl Display for Path { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let Path { - base, - sub_path, - version, - } = self; - - write!(f, "{}", base)?; - if let Some(version) = version { - write!(f, "@{}", version)?; - } - if let Some(sub) = sub_path { - write!(f, "#{}", sub)?; - } - - Ok(()) - } -} - -static PATH_PATTERN: Lazy = Lazy::new(|| { - Regex::new( - r"(?x) - (?P[\w\d:/_.-]+) - (?:@(?P[\w\d./-]+))? - (?:\#(?P[\w\d._/-]+))? - ", - ) - .unwrap() -}); - -impl FromStr for Path { - type Err = PathParseError; - fn from_str(s: &str) -> Result { - let captures = PATH_PATTERN.captures(s).ok_or(PathParseError)?; - - let base = captures["base"].to_string(); - let version = captures.name("version").map(|m| m.as_str().to_string()); - let sub_path = - captures.name("sub_path").map(|m| m.as_str().to_string()); - - Ok(Path { - base, - version, - sub_path, - }) + Document::parse(s) } } -impl Serialize for Path { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - self.to_string().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for Path { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = Cow::<'de, str>::deserialize(deserializer)?; - - s.parse().map_err(D::Error::custom) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Default)] -pub struct PathParseError; - -impl Display for PathParseError { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "Unable to parse the path") - } -} - -impl std::error::Error for PathParseError {} - /// A ML model which will be executed by the runtime. #[derive( Debug, Clone, PartialEq, + Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, @@ -311,7 +192,7 @@ impl std::error::Error for PathParseError {} pub struct ModelStage { /// The model to use, or a resource which specifies the model to use. #[schemars(required)] - pub model: ResourceOrString, + pub model: Path, /// Tensors to use as input to this model. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub inputs: Vec, @@ -327,6 +208,7 @@ pub struct ModelStage { Debug, Clone, PartialEq, + Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, @@ -344,11 +226,69 @@ pub struct ProcBlockStage { pub args: IndexMap, } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Path { + Uri(URI<'static>), + FileSystem(String), +} + +impl Serialize for Path { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Path { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer)?; + + s.parse().map_err(D::Error::custom) + } +} + +impl FromStr for Path { + type Err = URIError; + + fn from_str(s: &str) -> Result { + match URI::try_from(s) { + Ok(u) => Ok(Path::Uri(u.into_owned())), + Err(URIError::NotURI) => Ok(Path::FileSystem(s.to_string())), + Err(e) => Err(e), + } + } +} + +impl Display for Path { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Path::Uri(u) => u.fmt(f), + Path::FileSystem(p) => p.fmt(f), + } + } +} + +impl JsonSchema for Path { + fn schema_name() -> String { + String::from("Path") + } + + fn json_schema(gen: &mut schemars::gen::SchemaGenerator) -> Schema { + gen.subschema_for::() + } +} + /// A stage which reads inputs from the runtime. #[derive( Debug, Clone, PartialEq, + Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, @@ -368,6 +308,7 @@ pub struct CapabilityStage { Debug, Clone, PartialEq, + Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, @@ -384,7 +325,13 @@ pub struct OutStage { /// A stage in the Rune's pipeline. #[derive( - Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize, JsonSchema, + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + JsonSchema, )] #[serde(untagged, rename_all = "kebab-case")] pub enum Stage { @@ -430,11 +377,6 @@ impl Stage { } } - pub fn span(&self) -> Span { - // TODO: Get span from serde_yaml - Span::default() - } - pub fn args(&self) -> &IndexMap { match self { Stage::Model(m) => &m.args, @@ -443,6 +385,15 @@ impl Stage { Stage::Out(out) => &out.args, } } + + pub(crate) fn args_mut(&mut self) -> &mut IndexMap { + match self { + Stage::Model(m) => &mut m.args, + Stage::ProcBlock(p) => &mut p.args, + Stage::Capability(c) => &mut c.args, + Stage::Out(out) => &mut out.args, + } + } } /// Something that could be either a reference to a resource (`$resource`) @@ -454,7 +405,9 @@ pub enum ResourceOrString { } impl JsonSchema for ResourceOrString { - fn schema_name() -> std::string::String { "ResourceOrString".to_owned() } + fn schema_name() -> std::string::String { + "ResourceOrString".to_owned() + } fn json_schema(gen: &mut SchemaGenerator) -> Schema { let resource_name = gen.subschema_for::(); @@ -560,21 +513,27 @@ impl Display for ResourceOrString { } impl> From for ResourceOrString { - fn from(s: S) -> Self { ResourceOrString::String(s.into()) } + fn from(s: S) -> Self { + ResourceOrString::String(s.into()) + } } impl From for ResourceOrString { - fn from(name: ResourceName) -> Self { ResourceOrString::Resource(name) } + fn from(name: ResourceName) -> Self { + ResourceOrString::Resource(name) + } } /// A newtype around [`ResourceOrString`] which is used in each stage's `args` /// dictionary. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(transparent)] pub struct Argument(pub ResourceOrString); impl JsonSchema for Argument { - fn schema_name() -> std::string::String { "Argument".to_owned() } + fn schema_name() -> std::string::String { + "Argument".to_owned() + } fn json_schema(gen: &mut SchemaGenerator) -> Schema { let number = gen.subschema_for::(); @@ -587,13 +546,17 @@ impl JsonSchema for Argument { } impl> From for Argument { - fn from(value: T) -> Self { Argument(value.into()) } + fn from(value: T) -> Self { + Argument(value.into()) + } } impl Deref for Argument { type Target = ResourceOrString; - fn deref(&self) -> &Self::Target { &self.0 } + fn deref(&self) -> &Self::Target { + &self.0 + } } /// The element type and dimensions for a particular tensor. @@ -704,6 +667,7 @@ impl<'de> Deserialize<'de> for Input { Clone, Default, PartialEq, + Eq, serde::Serialize, serde::Deserialize, schemars::JsonSchema, @@ -718,13 +682,6 @@ pub struct ResourceDeclaration { pub ty: ResourceType, } -impl ResourceDeclaration { - pub fn span(&self) -> Span { - // TODO: Get span from serde_yaml - Span::default() - } -} - /// How the resource should be treated inside the Rune. #[derive( Debug, @@ -746,7 +703,9 @@ pub enum ResourceType { } impl Default for ResourceType { - fn default() -> Self { ResourceType::String } + fn default() -> Self { + ResourceType::String + } } /// A reference to some [`ResourceDeclaration`]. It typically looks like @@ -763,15 +722,10 @@ A reference to some [`ResourceDeclaration`]. It typically looks like "# ); -impl ResourceName { - pub fn span(&self) -> Span { - // TODO: Get span from serde_yaml - Span::default() - } -} - impl> From for ResourceName { - fn from(s: S) -> Self { ResourceName(s.into()) } + fn from(s: S) -> Self { + ResourceName(s.into()) + } } impl FromStr for ResourceName { @@ -793,7 +747,9 @@ impl FromStr for ResourceName { impl Deref for ResourceName { type Target = String; - fn deref(&self) -> &Self::Target { &self.0 } + fn deref(&self) -> &Self::Target { + &self.0 + } } impl Serialize for ResourceName { @@ -841,18 +797,32 @@ impl Display for ResourceName { Debug, Clone, PartialEq, + Eq, + Hash, serde::Serialize, serde::Deserialize, schemars::JsonSchema, )] #[schemars(transparent)] -pub struct Image(pub Path); +pub struct Image(String); + +impl Image { + pub fn runicos_base() -> Self { + Image(String::from("runicos/base")) + } +} impl FromStr for Image { - type Err = PathParseError; + type Err = std::convert::Infallible; fn from_str(s: &str) -> Result { - Path::from_str(s).map(Image) + Ok(Image(s.to_string())) + } +} + +impl Display for Image { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) } } @@ -862,6 +832,38 @@ mod tests { use super::*; + #[cfg(test)] + macro_rules! map { + // map-like + ($($k:ident : $v:expr),* $(,)?) => { + std::iter::Iterator::collect(IntoIterator::into_iter([ + $( + (String::from(stringify!($k)), $v) + ),* + ])) + }; + // set-like + ($($v:expr),* $(,)?) => { + std::iter::Iterator::collect(std::array::IntoIter::new([$($v,)*])) + }; + } + + #[cfg(test)] + macro_rules! ty { + ($type:ident [$($dim:expr),*]) => { + crate::parse::Type { + name: String::from(stringify!($type)), + dimensions: vec![ $($dim),*], + } + }; + ($type:ident) => { + crate::parse::Type { + name: String::from(stringify!($type)), + dimensions: vec![], + } + } + } + #[test] fn parse_normal_input_specifier() { let src = "audio"; @@ -884,59 +886,6 @@ mod tests { assert_eq!(got.to_string(), src); } - #[test] - fn parse_paths() { - let inputs = vec![ - ("asdf", Path::new("asdf", None, None)), - ("runicos/base", Path::new("runicos/base", None, None)), - ( - "runicos/base@0.1.2", - Path::new("runicos/base", None, Some(String::from("0.1.2"))), - ), - ( - "runicos/base@latest", - Path::new("runicos/base", None, Some(String::from("latest"))), - ), - ( - "https://github.com/hotg-ai/rune", - Path::new("https://github.com/hotg-ai/rune", None, None), - ), - ( - "https://github.com/hotg-ai/rune@2", - Path::new( - "https://github.com/hotg-ai/rune", - None, - Some(String::from("2")), - ), - ), - ( - "hotg-ai/rune@v1.2#proc_blocks/normalize", - Path::new( - "hotg-ai/rune", - "proc_blocks/normalize".to_string(), - "v1.2".to_string(), - ), - ), - // Note: GitHub provides these refs that you can use as well as the - // normal tags and commits - ( - "hotg-ai/proc-blocks@refs/heads/master#normalize", - Path::new( - "hotg-ai/proc-blocks", - "normalize".to_string(), - "refs/heads/master".to_string(), - ), - ), - ]; - - for (src, should_be) in inputs { - let got: Path = src.parse().unwrap(); - assert_eq!(got, should_be, "{}", src); - let round_tripped = got.to_string(); - assert_eq!(round_tripped, src); - } - } - #[test] fn parse_v1() { let src = "version: 1\nimage: asdf\npipeline: {}"; @@ -1084,7 +1033,7 @@ pipeline: hz: 16000 fft: - proc-block: "hotg-ai/rune#proc_blocks/fft" + proc-block: "git://github.com/hotg-ai/rune#proc_blocks/fft" inputs: - audio outputs: @@ -1100,7 +1049,7 @@ pipeline: dimensions: [6] label: - proc-block: "hotg-ai/rune#proc_blocks/ohv_label" + proc-block: "git://github.com/hotg-ai/rune#proc_blocks/ohv_label?tag=v0.11.3" inputs: - model outputs: @@ -1129,19 +1078,19 @@ pipeline: args: map! { hz: "16000".into() }, }), fft: Stage::ProcBlock(ProcBlockStage { - proc_block: "hotg-ai/rune#proc_blocks/fft".parse().unwrap(), + proc_block: "git://github.com/hotg-ai/rune#proc_blocks/fft".parse().unwrap(), inputs: vec!["audio".parse().unwrap()], outputs: vec![ty!(i8[1960])], args: IndexMap::new(), }), model: Stage::Model(ModelStage { - model: "./model.tflite".into(), + model: "./model.tflite".parse().unwrap(), inputs: vec!["fft".parse().unwrap()], outputs: vec![ty!(i8[6])], args: IndexMap::new(), }), label: Stage::ProcBlock(ProcBlockStage { - proc_block: "hotg-ai/rune#proc_blocks/ohv_label".parse().unwrap(), + proc_block: "git://github.com/hotg-ai/rune#proc_blocks/ohv_label?tag=v0.11.3".parse().unwrap(), inputs: vec!["model".parse().unwrap()], outputs: vec![Type { name: String::from("utf8"), dimensions: Vec::new() }], args: map! { @@ -1188,18 +1137,21 @@ pipeline: #[test] fn schema_is_in_sync_with_version_on_disk() { - let existing_schema = include_str!("../../runefile-schema.json"); - let should_be: serde_json::Value = - serde_json::from_str(existing_schema).unwrap(); + let filename = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("runefile-schema.json"); + let existing_schema = std::fs::read_to_string(&filename).unwrap(); + let existing_schema: serde_json::Value = + serde_json::from_str(&existing_schema).unwrap(); let schema = schemars::schema_for!(Document); + let current_schema = serde_json::to_value(&schema).unwrap(); - let schema = serde_json::to_value(&schema).unwrap(); - assert_eq!( - should_be, schema, - "The schema is out of sync. You probably need to run \"cargo \ - xtask update-schema\"", - ); + if existing_schema != current_schema { + let serialized = + serde_json::to_string_pretty(¤t_schema).unwrap(); + std::fs::write(&filename, serialized.as_bytes()).unwrap(); + panic!("The runefile-schema.json was out of date"); + } } #[track_caller] diff --git a/crates/compiler/src/phases.rs b/crates/compiler/src/phases.rs deleted file mode 100644 index 07cf0eaed4..0000000000 --- a/crates/compiler/src/phases.rs +++ /dev/null @@ -1,313 +0,0 @@ -use std::sync::Arc; - -use im::Vector; -use legion::{systems::Runnable, IntoQuery, Resources, World}; - -use crate::{ - codegen::{inputs::CodegenInputs, Codegen}, - compile::{CompilationResult, Compile}, - hooks::{Continuation, Ctx, Hooks}, - inputs::Inputs, - lowering::{self, Name}, - parse::parse, - type_check, BuildContext, Diagnostics, FeatureFlags, -}; - -/// Execute the `rune build` process. -pub fn build(ctx: BuildContext) -> (World, Resources) { - struct NopHooks; - impl Hooks for NopHooks {} - - build_with_hooks(ctx, FeatureFlags::production(), &mut NopHooks) -} - -/// Execute the `rune build` process, passing in custom [`Hooks`] which will -/// be fired after each phase. -pub fn build_with_hooks( - ctx: BuildContext, - features: FeatureFlags, - hooks: &mut dyn Hooks, -) -> (World, Resources) { - let mut db = Database::default(); - db.set_build_context(Arc::new(ctx.clone())); - db.set_feature_flags(features.clone()); - - let mut world = World::default(); - let mut res = Resources::default(); - - res.insert(ctx); - res.insert(features); - res.insert(Diagnostics::default()); - - if hooks.before_parse(&mut c(&mut world, &mut res)) - != Continuation::Continue - { - return (world, res); - } - - log::debug!("Beginning the \"parse\" phase"); - match parse(db.build_context().runefile.as_str()) { - Ok(d) => { - res.insert(d.clone().to_v1()); - }, - Err(e) => { - res.get_mut_or_default::() - .push(e.as_codespan_diagnostic()); - }, - } - - if hooks.after_parse(&mut c(&mut world, &mut res)) != Continuation::Continue - { - return (world, res); - } - - log::debug!("Beginning the \"lowering\" phase"); - lowering::phase().run(&mut world, &mut res); - - if hooks.after_lowering(&mut c(&mut world, &mut res)) - != Continuation::Continue - { - return (world, res); - } - - log::debug!("Beginning the \"type_check\" phase"); - type_check::phase().run(&mut world, &mut res); - - if hooks.after_type_checking(&mut c(&mut world, &mut res)) - != Continuation::Continue - { - return (world, res); - } - - log::debug!("Beginning the \"codegen\" phase"); - - update_db_before_codegen(&world, &mut db); - - let _files = db.files(); - - if hooks.after_codegen(&mut c(&mut world, &mut res)) - != Continuation::Continue - { - return (world, res); - } - - let result = db.build(); - res.insert(CompilationResult(result)); - - if hooks.after_compile(&mut c(&mut world, &mut res)) - != Continuation::Continue - { - return (world, res); - } - - (world, res) -} - -fn update_db_before_codegen(world: &World, db: &mut Database) { - let mut pb_names = Vector::new(); - <( - &Name, - &crate::lowering::ProcBlock, - &crate::lowering::Inputs, - &crate::lowering::Outputs, - )>::query() - .for_each(world, |(n, p, i, o)| { - pb_names.push_back(n.clone()); - db.set_node_inputs(n.clone(), i.clone()); - db.set_node_outputs(n.clone(), o.clone()); - db.set_proc_block_info(n.clone(), p.clone()); - }); - db.set_proc_block_names(pb_names); - - let mut model_names = Vector::new(); - <( - &Name, - &crate::lowering::Model, - &crate::lowering::ModelData, - &crate::lowering::Inputs, - &crate::lowering::Outputs, - )>::query() - .for_each(world, |(n, m, d, i, o)| { - model_names.push_back(n.clone()); - db.set_node_inputs(n.clone(), i.clone()); - db.set_node_outputs(n.clone(), o.clone()); - db.set_model_info(n.clone(), m.clone()); - db.set_model_data(n.clone(), d.clone()); - }); - db.set_model_names(model_names); -} - -#[derive(Default)] -#[salsa::database( - crate::codegen::inputs::CodegenInputsGroup, - crate::codegen::CodegenGroup, - crate::compile::CompileGroup, - crate::inputs::InputsGroup -)] -struct Database { - storage: salsa::Storage, -} - -impl salsa::Database for Database {} -impl crate::inputs::FileSystem for Database {} - -/// A group of operations which make up a single "phase" in the build process. -pub struct Phase(legion::systems::Builder); - -impl Phase { - pub(crate) fn new() -> Self { Phase(legion::Schedule::builder()) } - - pub(crate) fn with_setup( - mut setup: impl FnMut(&mut Resources) + 'static, - ) -> Self { - let mut phase = Phase::new(); - phase.0.add_thread_local_fn(move |_, res| setup(res)); - - phase - } - - pub(crate) fn and_then(mut self, run_system: F) -> Self - where - R: legion::systems::ParallelRunnable + 'static, - F: FnOnce() -> R, - { - self.0 - .add_system(TracingRunnable { - runnable: run_system(), - name: std::any::type_name::(), - }) - .flush(); - - self - } - - /// Execute the phase, updating the [`World`]. - pub fn run(&mut self, world: &mut World, resources: &mut Resources) { - self.0.build().execute(world, resources); - } -} - -/// A wrapper around some [`Runnable`] which logs whenever it starts. -struct TracingRunnable { - runnable: R, - name: &'static str, -} - -impl Runnable for TracingRunnable { - fn name(&self) -> Option<&legion::systems::SystemId> { - self.runnable.name() - } - - fn reads( - &self, - ) -> ( - &[legion::systems::ResourceTypeId], - &[legion::storage::ComponentTypeId], - ) { - self.runnable.reads() - } - - fn writes( - &self, - ) -> ( - &[legion::systems::ResourceTypeId], - &[legion::storage::ComponentTypeId], - ) { - self.runnable.writes() - } - - fn prepare(&mut self, world: &World) { self.runnable.prepare(world); } - - fn accesses_archetypes(&self) -> &legion::world::ArchetypeAccess { - self.runnable.accesses_archetypes() - } - - unsafe fn run_unsafe( - &mut self, - world: &World, - resources: &legion::systems::UnsafeResources, - ) { - let pretty_name = self - .name - .trim_start_matches(env!("CARGO_CRATE_NAME")) - .trim_end_matches("_system") - .trim_end_matches("::run") - .trim_matches(':'); - log::debug!("Starting the \"{}\" pass", pretty_name); - - self.runnable.run_unsafe(world, resources); - } - - fn command_buffer_mut( - &mut self, - world: legion::world::WorldId, - ) -> Option<&mut legion::systems::CommandBuffer> { - self.runnable.command_buffer_mut(world) - } -} - -fn c<'world, 'res>( - world: &'world mut World, - res: &'res mut Resources, -) -> Ctx<'world, 'res> { - Ctx { world, res } -} - -#[cfg(test)] -#[cfg(never)] -mod tests { - use indexmap::IndexMap; - - use super::*; - - #[test] - fn detect_pipeline_cycle() { - let src = r#" -image: runicos/base -version: 1 - -pipeline: - audio: - proc-block: "hotg-ai/rune#proc_blocks/fft" - inputs: - - model - outputs: - - type: i16 - dimensions: [16000] - - fft: - proc-block: "hotg-ai/rune#proc_blocks/fft" - inputs: - - audio - outputs: - - type: i8 - dimensions: [1960] - - model: - model: "./model.tflite" - inputs: - - fft - outputs: - - type: i8 - dimensions: [6] - "#; - let doc = Document::parse(src).unwrap(); - let mut diags = Diagnostics::new(); - - let _ = crate::analyse(doc, &mut diags); - - assert!(diags.has_errors()); - let errors: Vec<_> = diags - .iter_severity(codespan_reporting::diagnostic::Severity::Error) - .collect(); - assert_eq!(errors.len(), 1); - let diag = errors[0]; - assert_eq!(diag.message, "Cycle detected when checking \"audio\""); - assert!(diag.notes[0].contains("model")); - assert!(diag.notes[1].contains("fft")); - assert_eq!( - diag.notes[2], - "... which receives input from \"audio\", completing the cycle." - ); - } -} diff --git a/crates/compiler/src/serialize.rs b/crates/compiler/src/serialize.rs deleted file mode 100644 index 7410b4d8d7..0000000000 --- a/crates/compiler/src/serialize.rs +++ /dev/null @@ -1,63 +0,0 @@ -use legion::{ - serialize::{Canon, DeserializeNewWorld}, - storage::Component, - Registry, World, -}; -use serde::{de::DeserializeSeed, Deserializer, Serialize, Serializer}; - -pub(crate) trait RegistryExt { - fn register_with_type_name(&mut self) -> &mut Self - where - C: Component + serde::Serialize + for<'de> serde::Deserialize<'de>; -} - -impl RegistryExt for Registry { - fn register_with_type_name(&mut self) -> &mut Self - where - C: Component + serde::Serialize + for<'de> serde::Deserialize<'de>, - { - self.register::(std::any::type_name::().to_string()); - self - } -} - -/// Create a new [`Registry`] populated with the various component types used -/// in this crate. -pub fn registry() -> Registry { - let mut registry = Registry::new(); - - crate::lowering::register_components(&mut registry); - crate::type_check::register_components(&mut registry); - crate::codegen::register_components(&mut registry); - - registry -} - -pub fn serialize_world( - world: &World, - serializer: S, -) -> Result -where - S: Serializer, -{ - let registry = registry(); - let canon = Canon::default(); - - world - .as_serializable(legion::any(), ®istry, &canon) - .serialize(serializer) -} - -pub fn deserialize_world<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let registry = crate::serialize::registry(); - let canon = Canon::default(); - - DeserializeNewWorld { - world_deserializer: ®istry, - entity_serializer: &canon, - } - .deserialize(deserializer) -} diff --git a/crates/compiler/src/toolchain.rs b/crates/compiler/src/toolchain.rs deleted file mode 100644 index 655431866c..0000000000 --- a/crates/compiler/src/toolchain.rs +++ /dev/null @@ -1,27 +0,0 @@ -use cargo_toml::Value; - -/// Get a copy of the `rust-toolchain.toml` file used by the Rune project -/// itself. -pub fn rust_toolchain() -> Value { - toml::toml! { - [toolchain] - channel = "nightly-2022-02-27" - targets = ["wasm32-unknown-unknown"] - components = ["rustfmt"] - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn generated_toolchain_file_is_always_in_sync_with_repo() { - let original = include_str!("../../../rust-toolchain.toml"); - let original: Value = toml::from_str(original).unwrap(); - - let got = rust_toolchain(); - - assert_eq!(got, original); - } -} diff --git a/crates/compiler/src/type_check/check_for_loops.rs b/crates/compiler/src/type_check/check_for_loops.rs deleted file mode 100644 index 39a6f85eda..0000000000 --- a/crates/compiler/src/type_check/check_for_loops.rs +++ /dev/null @@ -1,114 +0,0 @@ -use std::collections::{HashSet, VecDeque}; - -use codespan::Span; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use indexmap::IndexMap; -use legion::{world::SubWorld, Entity, Query}; - -use crate::{ - lowering::{Name, Outputs}, - Diagnostics, -}; - -#[legion::system] -pub(crate) fn run( - world: &SubWorld, - #[resource] diags: &mut Diagnostics, - names: &mut Query<(&Name, &Span)>, - query: &mut Query<(Entity, &Outputs)>, -) { - // construct an adjacency graph where edges go from a node to its output. - // Note: use an IndexMap so ordering is deterministic - let mut outputs = IndexMap::new(); - query.for_each(world, |(&ent, out)| { - outputs.insert(ent, out.tensors.as_slice()); - }); - - if let Some(cycle) = next_cycle(&outputs) { - let cycle: Vec<_> = cycle - .iter() - .filter_map(|&ent| names.get(world, ent).ok()) - .collect(); - - let diag = cycle_detected_diagnostic(&cycle); - diags.push(diag); - } -} - -fn cycle_detected_diagnostic( - cycle: &[(&Name, &Span)], -) -> codespan_reporting::diagnostic::Diagnostic<()> { - let ((name, span), middle) = match cycle { - [first, middle @ ..] => (*first, middle), - _ => unreachable!("A cycle must have at least 2 items"), - }; - - let mut diag = Diagnostic::error() - .with_message(format!("Cycle detected when checking \"{}\"", name)); - - diag = diag.with_labels(vec![Label::primary((), *span)]); - - let mut notes = Vec::new(); - - for (name, _) in middle { - let msg = format!("... which passes data to \"{}\"...", name); - notes.push(msg); - } - - let closing_message = format!( - "... which passes data to \"{}\", completing the cycle.", - name - ); - notes.push(closing_message); - - diag.with_notes(notes) -} - -fn next_cycle(outputs: &IndexMap) -> Option> { - // https://www.geeksforgeeks.org/detect-cycle-in-a-graph/ - let mut stack = VecDeque::new(); - let mut visited = HashSet::new(); - - for ent in outputs.keys().copied() { - if detect_cycles(ent, outputs, &mut visited, &mut stack) { - return Some(stack.into()); - } - } - - None -} - -fn detect_cycles( - ent: Entity, - outputs: &IndexMap, - visited: &mut HashSet, - stack: &mut VecDeque, -) -> bool { - if stack.contains(&ent) { - // We've detected a cycle, remove everything before our id so the stack - // is left just containing the cycle - while stack.front() != Some(&ent) { - stack.pop_front(); - } - - return true; - } else if visited.contains(&ent) { - return false; - } - - visited.insert(ent); - stack.push_back(ent); - - let outgoing_node = outputs.get(&ent).copied().unwrap_or_default(); - - for &outgoing_node in outgoing_node { - if detect_cycles(outgoing_node, outputs, visited, stack) { - return true; - } - } - - let got = stack.pop_back(); - debug_assert_eq!(got, Some(ent)); - - false -} diff --git a/crates/compiler/src/type_check/components.rs b/crates/compiler/src/type_check/components.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/crates/compiler/src/type_check/components.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/compiler/src/type_check/mod.rs b/crates/compiler/src/type_check/mod.rs index a5d4699aa8..5fa6349073 100644 --- a/crates/compiler/src/type_check/mod.rs +++ b/crates/compiler/src/type_check/mod.rs @@ -1,18 +1,5 @@ -//! The type checking phase. +//! The Rune compiler's type checker. -mod check_for_loops; -mod components; -mod model_args_are_consumed; +mod types; -pub use components::*; -use legion::Registry; - -use crate::phases::Phase; - -pub fn phase() -> Phase { - Phase::new() - .and_then(check_for_loops::run_system) - .and_then(model_args_are_consumed::run_system) -} - -pub(crate) fn register_components(_registry: &mut Registry) {} +pub use types::*; diff --git a/crates/compiler/src/type_check/model_args_are_consumed.rs b/crates/compiler/src/type_check/model_args_are_consumed.rs deleted file mode 100644 index 3cbbc30580..0000000000 --- a/crates/compiler/src/type_check/model_args_are_consumed.rs +++ /dev/null @@ -1,39 +0,0 @@ -use codespan::Span; -use codespan_reporting::diagnostic::{Diagnostic, Label}; -use legion::{world::SubWorld, Query}; - -use crate::{ - lowering::{Model, Name}, - Diagnostics, -}; - -/// Check that all model arguments were consumed during the lowering process, -/// emitting a warning for any that weren't. -#[legion::system] -pub(crate) fn run( - world: &SubWorld, - #[resource] diags: &mut Diagnostics, - models: &mut Query<(&Name, &Span, &Model)>, -) { - models.for_each(world, |(n, s, m)| { - if !m.args.is_empty() { - let unused_args: Vec<_> = - m.args.keys().map(|s| s.as_str()).collect(); - diags.push(unused_model_arguments_diagnostic(n, *s, &unused_args)); - } - }); -} - -fn unused_model_arguments_diagnostic( - name: &Name, - span: Span, - unused_args: &[&str], -) -> Diagnostic<()> { - Diagnostic::warning() - .with_message(format!( - "Unused arguments for {}: {}", - name, - unused_args.join(", ") - )) - .with_labels(vec![Label::primary((), span)]) -} diff --git a/crates/compiler-2/src/type_check/types.rs b/crates/compiler/src/type_check/types.rs similarity index 100% rename from crates/compiler-2/src/type_check/types.rs rename to crates/compiler/src/type_check/types.rs diff --git a/crates/compiler/tests/existing_runefiles.rs b/crates/compiler/tests/existing_runefiles.rs deleted file mode 100644 index 2c0207fc9e..0000000000 --- a/crates/compiler/tests/existing_runefiles.rs +++ /dev/null @@ -1,184 +0,0 @@ -use codespan_reporting::{ - files::SimpleFile, - term::{termcolor::Buffer, Config}, -}; -use hotg_rune_compiler::{ - codegen::RuneVersion, - hooks::{ - AfterCodegenContext, AfterTypeCheckingContext, Continuation, Hooks, - }, - parse::Document, - BuildContext, Diagnostics, FeatureFlags, Verbosity, -}; -use jsonschema::JSONSchema; -use serde_json::Value; - -macro_rules! parse_and_analyse { - ($example:ident) => { - mod $example { - use super::*; - - const PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../examples/", - stringify!($example), - ); - const SRC: &str = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../examples/", - stringify!($example), - "/Runefile.yml" - )); - - #[test] - fn parse() { let _ = Document::parse(SRC).unwrap(); } - - #[derive(Debug, Copy, Clone, PartialEq)] - enum Phase { - TypeCheck, - Codegen, - } - - struct AbortAfterPhase { - diags: Diagnostics, - phase: Phase, - } - - impl AbortAfterPhase { - fn new(phase: Phase) -> Self { - AbortAfterPhase { - phase, - diags: Diagnostics::new(), - } - } - - fn maybe_abort( - &mut self, - phase: Phase, - diags: &Diagnostics, - ) -> Continuation { - if phase == self.phase { - for diag in diags.iter() { - self.diags.push(diag.clone()); - } - Continuation::Halt - } else { - Continuation::Continue - } - } - } - - impl Hooks for AbortAfterPhase { - fn after_type_checking( - &mut self, - ctx: &mut dyn AfterTypeCheckingContext, - ) -> Continuation { - self.maybe_abort(Phase::TypeCheck, &ctx.diagnostics()) - } - - fn after_codegen( - &mut self, - ctx: &mut dyn AfterCodegenContext, - ) -> Continuation { - self.maybe_abort(Phase::Codegen, &ctx.diagnostics()) - } - } - - fn handle_diagnostics( - file: &SimpleFile<&'static str, &'static str>, - diags: &Diagnostics, - ) { - let mut writer = Buffer::no_color(); - let config = Config::default(); - - for diag in diags { - codespan_reporting::term::emit( - &mut writer, - &config, - file, - diag, - ) - .unwrap(); - } - - if diags.has_errors() { - panic!("{}", String::from_utf8_lossy(writer.as_slice())); - } - } - - fn build_context() -> BuildContext { - BuildContext { - name: stringify!($example).to_string(), - runefile: SRC.to_string(), - working_directory: PATH.into(), - current_directory: PATH.into(), - optimized: false, - verbosity: Verbosity::Normal, - rune_version: Some(RuneVersion { - version: env!("CARGO_PKG_VERSION").to_string(), - }), - } - } - - #[test] - fn analyse() { - let file = SimpleFile::new("Runefile", SRC); - let ctx = build_context(); - let mut hooks = AbortAfterPhase::new(Phase::TypeCheck); - - hotg_rune_compiler::build_with_hooks( - ctx, - FeatureFlags::development(), - &mut hooks, - ); - - handle_diagnostics(&file, &hooks.diags); - } - - #[test] - fn codegen() { - let file = SimpleFile::new("Runefile", SRC); - let ctx = build_context(); - let mut hooks = AbortAfterPhase::new(Phase::Codegen); - - hotg_rune_compiler::build_with_hooks( - ctx, - FeatureFlags::development(), - &mut hooks, - ); - - handle_diagnostics(&file, &hooks.diags); - } - - #[test] - fn validate_against_yaml_schema() { - let document: Value = serde_yaml::from_str(SRC).unwrap(); - - let schema = schemars::schema_for!(Document); - let schema = serde_json::to_value(&schema).unwrap(); - - let compiled_schema = - JSONSchema::options().compile(&schema).unwrap(); - - let result = compiled_schema.validate(&document); - if let Err(errors) = result { - for error in errors { - println!("Validation error: {}", error); - println!("Instance path: {}", error.instance_path); - println!(); - } - - panic!("Validation failed"); - } - } - } - }; -} - -parse_and_analyse!(debugging); -parse_and_analyse!(gesture); -parse_and_analyse!(microspeech); -parse_and_analyse!(noop); -parse_and_analyse!(person_detection); -parse_and_analyse!(sine); -parse_and_analyse!(style_transfer); From aa43e3db1829c4f3aa3e223a90ebee50d1ee0a27 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 29 Apr 2022 12:17:19 +0800 Subject: [PATCH 27/84] Added an example compiler binary --- Cargo.lock | 2 + crates/compiler/Cargo.toml | 2 + crates/compiler/README.md | 19 ++++++- crates/compiler/examples/compile.rs | 84 +++++++++++++++++++++++++++++ examples/sine/Runefile.yml | 2 +- 5 files changed, 107 insertions(+), 2 deletions(-) mode change 120000 => 100644 crates/compiler/README.md create mode 100644 crates/compiler/examples/compile.rs diff --git a/Cargo.lock b/Cargo.lock index bc7bbbe327..eab1b84e52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1322,6 +1322,7 @@ dependencies = [ "once_cell", "pretty_assertions 1.2.1", "regex", + "reqwest", "salsa", "schemars", "serde", @@ -1329,6 +1330,7 @@ dependencies = [ "serde_yaml", "thiserror", "tracing", + "tracing-subscriber", "tracing-test", "uriparse", "zip 0.6.2", diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index f9a5edf471..b10f8b0fff 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -26,4 +26,6 @@ zip = "0.6.2" insta = "1.14.0" jsonschema = "0.15.2" pretty_assertions = "1.2.1" +reqwest = "0.11.10" +tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } tracing-test = "0.2.1" diff --git a/crates/compiler/README.md b/crates/compiler/README.md deleted file mode 120000 index fe84005413..0000000000 --- a/crates/compiler/README.md +++ /dev/null @@ -1 +0,0 @@ -../../README.md \ No newline at end of file diff --git a/crates/compiler/README.md b/crates/compiler/README.md new file mode 100644 index 0000000000..1110c991f0 --- /dev/null +++ b/crates/compiler/README.md @@ -0,0 +1,18 @@ +# The Rune Compiler + +A compiler that compiles your data processing pipeline into a portable +WebAssembly binary. + +## Architecture + +The Rune compiler is base around [Salsa][salsa], a library for incremental +computation. This lets us phrase the compilation process as a series of queries +(essentially, pure functions) which can be aggressively cached based on +dependency analysis. + +These series of queries are broken up into a couple submodules, + +- [`parse`] - Parse a Runefile written in the YAML format +- [`codegen`] - Generate the final Rune binary + +[salsa]: https://github.com/salsa-rs/salsa diff --git a/crates/compiler/examples/compile.rs b/crates/compiler/examples/compile.rs new file mode 100644 index 0000000000..94551aa042 --- /dev/null +++ b/crates/compiler/examples/compile.rs @@ -0,0 +1,84 @@ +use std::path::PathBuf; + +use hotg_rune_compiler::{ + codegen::{Codegen, CodegenStorage}, + im::Vector, + parse::{Frontend, FrontendStorage}, + BuildConfig, Environment, EnvironmentStorage, FileSystem, ReadError, +}; +use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; +use uriparse::{Scheme, URI}; + +fn main() { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::default() + .add_directive("hotg_rune_compiler=debug".parse().unwrap()) + .add_directive("compile=debug".parse().unwrap()), + ) + .with_span_events(FmtSpan::CLOSE) + .init(); + + let runefile = std::env::args() + .nth(1) + .expect("Usage: validate-runefile-schema "); + let runefile = PathBuf::from(runefile); + + let src = std::fs::read_to_string(&runefile).unwrap(); + let parent = runefile.parent().unwrap().to_path_buf(); + + let mut db = Database::default(); + db.set_src(src.into()); + db.set_config(BuildConfig { + current_directory: parent.clone(), + }); + + let archive = db.rune_archive().unwrap(); + + let name = parent.file_name().unwrap(); + std::fs::write(parent.join(name).with_extension("zip"), &*archive).unwrap(); +} + +#[derive(Default)] +#[salsa::database(FrontendStorage, EnvironmentStorage, CodegenStorage)] +struct Database { + storage: salsa::Storage, +} + +impl salsa::Database for Database {} + +// The parsing process requires you to load proc-blocks and read files. You +// can satisfy these dependencies by implementing the corresponding traits. + +impl FileSystem for Database { + fn read(&self, uri: &URI<'_>) -> Result, ReadError> { + let _span = tracing::info_span!("read", %uri).entered(); + + match uri.scheme() { + Scheme::File => { + let filename = uri.path().to_string(); + let contents = + std::fs::read(&filename).map_err(ReadError::other)?; + + tracing::info!(bytes_read = contents.len(), %filename, "Read a file from disk"); + + Ok(contents.into()) + }, + Scheme::HTTP | Scheme::HTTPS => { + tracing::info!("Downloading"); + let response = reqwest::blocking::get(uri.to_string()) + .and_then(|r| r.error_for_status()) + .map_err(ReadError::other)?; + + let body = response.bytes().map_err(ReadError::other)?; + tracing::debug!(bytes_read = body.len(), "Download complete"); + + Ok(body.to_vec().into()) + }, + Scheme::Unregistered(s) if s.as_str() == "wapm" => { + Ok(Vector::default()) + }, + _ => unimplemented!(), + } + } +} diff --git a/examples/sine/Runefile.yml b/examples/sine/Runefile.yml index 5bcf59bbe8..e6ae56628f 100644 --- a/examples/sine/Runefile.yml +++ b/examples/sine/Runefile.yml @@ -11,7 +11,7 @@ pipeline: - 1 - 1 mod360: - proc-block: "wapm://hotg-ai/modulo/v0.11.3" + proc-block: "https://func.hotg.ai/function/sbfs/pb/argmax.wasm" inputs: - rand outputs: From 8f8b98bbeca4be212b742a401b40b500fa598b6c Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 30 Apr 2022 06:29:02 +0800 Subject: [PATCH 28/84] Errors should be thread-safe --- crates/compiler/src/codegen/query.rs | 23 ++++++------ crates/compiler/src/lib.rs | 2 ++ crates/compiler/src/parse/query.rs | 53 +++++++++++++--------------- 3 files changed, 37 insertions(+), 41 deletions(-) diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index 1b500a5aae..659ff0aa5c 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -1,5 +1,4 @@ use std::{ - error::Error, io::{Cursor, Seek, Write}, sync::Arc, }; @@ -25,18 +24,16 @@ pub trait Codegen: Frontend { /// pointing to "local" (according to the ZIP archive generated by /// [`Codegen::rune_archive()`]) files. #[salsa::dependencies] - fn self_contained_runefile( - &self, - ) -> Result, Arc>; + fn self_contained_runefile(&self) -> Result, crate::Error>; #[salsa::dependencies] - fn rune_archive(&self) -> Result, Arc>; + fn rune_archive(&self) -> Result, crate::Error>; } #[tracing::instrument(skip(db))] fn self_contained_runefile( db: &dyn Codegen, -) -> Result, Arc> { +) -> Result, crate::Error> { let mut doc = db.parse()?; let d = Arc::make_mut(&mut doc); @@ -51,7 +48,7 @@ fn self_contained_runefile( fn patch_arguments( db: &dyn Codegen, stages: &mut IndexMap, -) -> Result<(), Arc> { +) -> Result<(), crate::Error> { for (stage_name, stage) in stages { for (arg_name, arg_value) in stage.args_mut() { if let Argument(ResourceOrString::Resource(ResourceName(res))) = @@ -65,7 +62,7 @@ fn patch_arguments( "Patched an argument", ); let s = std::str::from_utf8(&value) - .map_err(|e| Arc::new(e) as Arc)?; + .map_err(|e| Arc::new(e) as crate::Error)?; *arg_value = Argument(ResourceOrString::String(s.to_string())); } } @@ -77,7 +74,7 @@ fn patch_arguments( #[tracing::instrument(skip(stages))] fn patch_paths( stages: &mut IndexMap, -) -> Result<(), Arc> { +) -> Result<(), crate::Error> { for (name, stage) in stages { match stage { Stage::Model(m) => { @@ -100,7 +97,7 @@ fn patch_paths( #[tracing::instrument(skip(resources))] fn patch_resources( resources: &mut IndexMap, -) -> Result<(), Arc> { +) -> Result<(), crate::Error> { for (name, decl) in resources { decl.inline = None; let path = format!("resources/{name}"); @@ -112,7 +109,7 @@ fn patch_resources( } #[tracing::instrument(skip(db))] -fn rune_archive(db: &dyn Codegen) -> Result, Arc> { +fn rune_archive(db: &dyn Codegen) -> Result, crate::Error> { let runefile = db.self_contained_runefile()?; let runefile = serde_yaml::to_string(&*runefile) .expect("Serializing to YAML should never fail"); @@ -122,7 +119,7 @@ fn rune_archive(db: &dyn Codegen) -> Result, Arc> { generate_archive(&runefile, &proc_blocks, &models, &resources) .map(Vector::from) - .map_err(|e| Arc::new(e) as Arc) + .map_err(|e| Arc::new(e) as crate::Error) } fn generate_archive( @@ -170,7 +167,7 @@ fn write_to_directory( #[cfg(test)] mod tests { - use std::{path::Path}; + use std::path::Path; use tracing_test::traced_test; use uriparse::{Scheme, URI}; use zip::ZipArchive; diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 04a46577cf..4130d0d5eb 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -16,3 +16,5 @@ pub use crate::{ filesystem::{FileSystem, ReadError}, im::Text, }; + +pub type Error = std::sync::Arc; diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index 2519bba86f..fab9f86bfd 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, error::Error, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc}; use uriparse::{URIBuilder, URIError, URI}; @@ -74,7 +74,7 @@ pub trait Frontend: Environment + FileSystem { /// Parse the [`Frontend::src()`] into a [`DocumentV1`]. #[salsa::dependencies] - fn parse(&self) -> Result, Arc>; + fn parse(&self) -> Result, crate::Error>; /// A low-level query for parsing a YAML file. #[salsa::dependencies] @@ -83,26 +83,25 @@ pub trait Frontend: Environment + FileSystem { /// Get a resource's value by either returning the value as-is (for /// `inline` resources) or reading the file from the filesystem. #[salsa::dependencies] - fn resource_value(&self, name: Text) -> Result, Arc>; + fn resource_value(&self, name: Text) -> Result, crate::Error>; #[salsa::dependencies] - fn resource_values( - &self, - ) -> Result>, Arc>; + fn resource_values(&self) + -> Result>, crate::Error>; #[salsa::dependencies] - fn proc_block(&self, name: Text) -> Result, Arc>; + fn proc_block(&self, name: Text) -> Result, crate::Error>; /// Get the binary data for each proc-block in this Rune, ignoring any which /// may have failed to load. #[salsa::dependencies] - fn proc_blocks(&self) -> Result>, Arc>; + fn proc_blocks(&self) -> Result>, crate::Error>; #[salsa::dependencies] - fn model_file(&self, name: Text) -> Result, Arc>; + fn model_file(&self, name: Text) -> Result, crate::Error>; #[salsa::dependencies] - fn model_files(&self) -> Result>, Arc>; + fn model_files(&self) -> Result>, crate::Error>; } #[tracing::instrument(skip(src), err)] @@ -116,17 +115,17 @@ fn parse_runefile( } #[tracing::instrument(skip(db))] -fn parse(db: &dyn Frontend) -> Result, Arc> { +fn parse(db: &dyn Frontend) -> Result, crate::Error> { db.parse_runefile(db.src()) .map(|d| Arc::new(Document::clone(&d).to_v1())) - .map_err(|e| Arc::new(e) as Arc) + .map_err(|e| Arc::new(e) as crate::Error) } #[tracing::instrument(skip(db), err)] fn resource_value( db: &dyn Frontend, name: Text, -) -> Result, Arc> { +) -> Result, crate::Error> { let doc = db.parse()?; let ResourceDeclaration { inline, path, .. } = @@ -134,7 +133,7 @@ fn resource_value( Arc::new(NotFound { name, item_type: ItemType::Resource, - }) as Arc + }) as crate::Error })?; match (inline, path) { @@ -151,7 +150,7 @@ fn resource_value( #[tracing::instrument(skip(db))] fn proc_blocks( db: &dyn Frontend, -) -> Result>, Arc> { +) -> Result>, crate::Error> { let doc = db.parse()?; let mut proc_blocks = BTreeMap::default(); @@ -170,7 +169,7 @@ fn proc_blocks( fn proc_block( db: &dyn Frontend, name: Text, -) -> Result, Arc> { +) -> Result, crate::Error> { let doc = db.parse()?; let stage = doc @@ -180,7 +179,7 @@ fn proc_block( name, item_type: ItemType::ProcBlock, }) - .map_err(|e| Arc::new(e) as Arc)?; + .map_err(|e| Arc::new(e) as crate::Error)?; if let Stage::ProcBlock(ProcBlockStage { proc_block, .. }) = stage { read(db, &proc_block) @@ -192,7 +191,7 @@ fn proc_block( #[tracing::instrument(skip(db), err)] fn resource_values( db: &dyn Frontend, -) -> Result>, Arc> { +) -> Result>, crate::Error> { let doc = db.parse()?; doc.resources @@ -209,7 +208,7 @@ fn resource_values( fn model_file( db: &dyn Frontend, name: Text, -) -> Result, Arc> { +) -> Result, crate::Error> { let doc = db.parse()?; let stage = doc @@ -219,15 +218,15 @@ fn model_file( name: name.clone(), item_type: ItemType::Model, }) - .map_err(|e| Arc::new(e) as Arc)?; + .map_err(|e| Arc::new(e) as crate::Error)?; - let err = move |actual: ItemType| -> Result, Arc> { + let err = move |actual: ItemType| -> Result, crate::Error> { let e = WrongItemType { expected: ItemType::Model, actual, name, }; - Err(Arc::new(e) as Arc) + Err(Arc::new(e) as crate::Error) }; match stage { @@ -241,7 +240,7 @@ fn model_file( #[tracing::instrument(skip(db))] fn model_files( db: &dyn Frontend, -) -> Result>, Arc> { +) -> Result>, crate::Error> { let doc = db.parse()?; let mut models = BTreeMap::default(); @@ -256,12 +255,10 @@ fn model_files( Ok(models.into()) } -fn read(db: &dyn Frontend, path: &Path) -> Result, Arc> { +fn read(db: &dyn Frontend, path: &Path) -> Result, crate::Error> { file_uri(db, path) - .map_err(|e| Arc::new(e) as Arc) - .and_then(|uri| { - db.read(&uri).map_err(|e| Arc::new(e) as Arc) - }) + .map_err(|e| Arc::new(e) as crate::Error) + .and_then(|uri| db.read(&uri).map_err(|e| Arc::new(e) as crate::Error)) } fn file_uri(db: &dyn Frontend, path: &Path) -> Result, URIError> { From 455d77d893759dfd50975e6666d57ff8862bee21 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 30 Apr 2022 07:03:33 +0800 Subject: [PATCH 29/84] Made all instrumented queries log at debug level --- crates/compiler/src/codegen/query.rs | 12 ++++++------ crates/compiler/src/parse/query.rs | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index 659ff0aa5c..070ac984d5 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -30,7 +30,7 @@ pub trait Codegen: Frontend { fn rune_archive(&self) -> Result, crate::Error>; } -#[tracing::instrument(skip(db))] +#[tracing::instrument(level = "debug", skip(db))] fn self_contained_runefile( db: &dyn Codegen, ) -> Result, crate::Error> { @@ -44,7 +44,7 @@ fn self_contained_runefile( Ok(doc) } -#[tracing::instrument(skip(db, stages))] +#[tracing::instrument(level = "debug", skip(db, stages))] fn patch_arguments( db: &dyn Codegen, stages: &mut IndexMap, @@ -71,7 +71,7 @@ fn patch_arguments( Ok(()) } -#[tracing::instrument(skip(stages))] +#[tracing::instrument(level = "debug", skip(stages))] fn patch_paths( stages: &mut IndexMap, ) -> Result<(), crate::Error> { @@ -94,7 +94,7 @@ fn patch_paths( Ok(()) } -#[tracing::instrument(skip(resources))] +#[tracing::instrument(level = "debug", skip(resources))] fn patch_resources( resources: &mut IndexMap, ) -> Result<(), crate::Error> { @@ -108,7 +108,7 @@ fn patch_resources( Ok(()) } -#[tracing::instrument(skip(db))] +#[tracing::instrument(level = "debug", skip(db))] fn rune_archive(db: &dyn Codegen) -> Result, crate::Error> { let runefile = db.self_contained_runefile()?; let runefile = serde_yaml::to_string(&*runefile) @@ -151,7 +151,7 @@ fn generate_archive( Ok(buffer) } -#[tracing::instrument(skip(writer, data))] +#[tracing::instrument(level = "debug", skip(writer, data))] fn write_to_directory( writer: &mut ZipWriter, directory: &str, diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index fab9f86bfd..ec35075021 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -104,7 +104,7 @@ pub trait Frontend: Environment + FileSystem { fn model_files(&self) -> Result>, crate::Error>; } -#[tracing::instrument(skip(src), err)] +#[tracing::instrument(level = "debug", skip(src), err)] fn parse_runefile( _: &dyn Frontend, src: Text, @@ -114,14 +114,14 @@ fn parse_runefile( .map_err(|e| ParseFailed { error: Arc::new(e) }) } -#[tracing::instrument(skip(db))] +#[tracing::instrument(level = "debug", skip(db))] fn parse(db: &dyn Frontend) -> Result, crate::Error> { db.parse_runefile(db.src()) .map(|d| Arc::new(Document::clone(&d).to_v1())) .map_err(|e| Arc::new(e) as crate::Error) } -#[tracing::instrument(skip(db), err)] +#[tracing::instrument(level = "debug", skip(db), err)] fn resource_value( db: &dyn Frontend, name: Text, @@ -147,7 +147,7 @@ fn resource_value( } } -#[tracing::instrument(skip(db))] +#[tracing::instrument(level = "debug", skip(db))] fn proc_blocks( db: &dyn Frontend, ) -> Result>, crate::Error> { @@ -165,7 +165,7 @@ fn proc_blocks( Ok(proc_blocks.into()) } -#[tracing::instrument(skip(db), err)] +#[tracing::instrument(level = "debug", skip(db), err)] fn proc_block( db: &dyn Frontend, name: Text, @@ -188,7 +188,7 @@ fn proc_block( } } -#[tracing::instrument(skip(db), err)] +#[tracing::instrument(level = "debug", skip(db), err)] fn resource_values( db: &dyn Frontend, ) -> Result>, crate::Error> { @@ -204,7 +204,7 @@ fn resource_values( .collect() } -#[tracing::instrument(skip(db), err)] +#[tracing::instrument(level = "debug", skip(db), err)] fn model_file( db: &dyn Frontend, name: Text, @@ -237,7 +237,7 @@ fn model_file( } } -#[tracing::instrument(skip(db))] +#[tracing::instrument(level = "debug", skip(db))] fn model_files( db: &dyn Frontend, ) -> Result>, crate::Error> { From 67ae98de99447f0e1f761a9c4f791451d6d3892b Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 30 Apr 2022 07:06:47 +0800 Subject: [PATCH 30/84] Renamed ReadError::UnknownScheme to UnsupportedScheme --- crates/compiler/src/filesystem.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/compiler/src/filesystem.rs b/crates/compiler/src/filesystem.rs index 67260c88fe..f01bfc641e 100644 --- a/crates/compiler/src/filesystem.rs +++ b/crates/compiler/src/filesystem.rs @@ -13,10 +13,8 @@ pub trait FileSystem { #[derive(Debug, Clone, thiserror::Error)] pub enum ReadError { - #[error("Unknown scheme, \"{}\"", scheme)] - UnknownScheme { scheme: Text }, - #[error("This operation isn't supported")] - NotSupported, + #[error("The \"{}\" scheme isn't supported", scheme)] + UnsupportedScheme { scheme: Text }, #[error(transparent)] Other(Arc), } From 96197af8543a3562936ce9d4bc3ae0023d3d3a93 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 10 May 2022 21:49:14 +0800 Subject: [PATCH 31/84] Switched "rune build" over to using the compiler from crates.io --- .gitignore | 1 - Cargo.lock | 163 ++++++++++++++-- crates/compiler/src/codegen/query.rs | 8 +- crates/compiler/src/config.rs | 9 + crates/compiler/src/lib.rs | 2 +- crates/rune-cli/Cargo.toml | 4 +- crates/rune-cli/src/build/mod.rs | 89 +++++++++ .../src/{build.rs => build/rune_v0.rs} | 180 ++++++------------ crates/rune-cli/src/inspect/proc_block.rs | 5 - crates/rune-cli/tests/integration.rs | 13 +- 10 files changed, 321 insertions(+), 153 deletions(-) create mode 100644 crates/rune-cli/src/build/mod.rs rename crates/rune-cli/src/{build.rs => build/rune_v0.rs} (51%) diff --git a/.gitignore b/.gitignore index 3a2972c887..15757c3e10 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,3 @@ examples/.DS_Store *.png tarpaulin-report.html .ipynb_checkpoints/ -build/ diff --git a/Cargo.lock b/Cargo.lock index eab1b84e52..e766da64e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,12 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "atomic_refcell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" + [[package]] name = "atty" version = "0.2.14" @@ -402,7 +408,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e3973b165dc0f435831a9e426de67e894de532754ff7a3f307c03ee5dec7dc" dependencies = [ "clap", - "heck", + "heck 0.3.3", "indexmap", "log", "proc-macro2", @@ -496,12 +502,23 @@ dependencies = [ "cc", ] +[[package]] +name = "codespan" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3362992a0d9f1dd7c3d0e89e0ab2bb540b7a95fea8cd798090e758fda2899b5e" +dependencies = [ + "codespan-reporting", + "serde", +] + [[package]] name = "codespan-reporting" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ + "serde", "termcolor", "unicode-width", ] @@ -887,6 +904,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + [[package]] name = "dyn-clone" version = "1.0.5" @@ -968,6 +991,15 @@ dependencies = [ "termcolor", ] +[[package]] +name = "erased-serde" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad132dd8d0d0b546348d7d86cb3191aad14b34e5f979781fc005c80d4ac67ffd" +dependencies = [ + "serde", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1256,6 +1288,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1288,9 +1326,9 @@ dependencies = [ "dirs", "dotenv", "env_logger", - "hotg-rune-compiler", - "hotg-rune-core", - "hotg-rune-proc-blocks", + "hotg-rune-compiler 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hotg-rune-core 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hotg-rune-proc-blocks 0.11.3", "hotg-rune-runtime", "hotg-runecoral", "hound", @@ -1336,6 +1374,35 @@ dependencies = [ "zip 0.6.2", ] +[[package]] +name = "hotg-rune-compiler" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbbee35125a9021f5d1406748c8a93145f36e171f5793ae5f9c9d219c14dc4d9" +dependencies = [ + "atomic_refcell", + "cargo_toml", + "codespan", + "codespan-reporting", + "heck 0.4.0", + "hotg-rune-core 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hotg-rune-proc-blocks 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "indoc", + "legion", + "log", + "once_cell", + "proc-macro2", + "quote", + "regex", + "schemars", + "serde", + "serde_json", + "serde_yaml", + "toml", + "zip 0.5.13", +] + [[package]] name = "hotg-rune-core" version = "0.11.3" @@ -1344,6 +1411,16 @@ dependencies = [ "serde", ] +[[package]] +name = "hotg-rune-core" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575d4d19eed25d6960b9bdcab2b307f00b67f274105ab8bfa2c6debb38bf6075" +dependencies = [ + "log", + "serde", +] + [[package]] name = "hotg-rune-integration-tests" version = "0.0.0" @@ -1363,7 +1440,7 @@ name = "hotg-rune-proc-block-macros" version = "0.11.3" dependencies = [ "difference", - "hotg-rune-core", + "hotg-rune-core 0.11.3", "pretty_assertions 1.2.1", "proc-macro2", "quote", @@ -1377,19 +1454,29 @@ name = "hotg-rune-proc-blocks" version = "0.11.3" dependencies = [ "difference", - "hotg-rune-core", + "hotg-rune-core 0.11.3", "hotg-rune-proc-block-macros", "pretty_assertions 1.2.1", "serde", ] +[[package]] +name = "hotg-rune-proc-blocks" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b188032354021389a1c57233ff3f2f42341af703cff571d9551bc7f3eb4ddd5c" +dependencies = [ + "hotg-rune-core 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", +] + [[package]] name = "hotg-rune-runtime" version = "0.11.3" dependencies = [ "anyhow", "csv", - "hotg-rune-core", + "hotg-rune-core 0.11.3", "hotg-runecoral", "hound", "image", @@ -1421,7 +1508,7 @@ name = "hotg-runicos-base-wasm" version = "0.11.3" dependencies = [ "dlmalloc", - "hotg-rune-core", + "hotg-rune-core 0.11.3", "log", "serde", "serde-json-core", @@ -1573,6 +1660,12 @@ dependencies = [ "serde", ] +[[package]] +name = "indoc" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05a0bd019339e5d968b37855180087b7b9d512c5046fbd244cf8c95687927d6e" + [[package]] name = "insta" version = "1.14.0" @@ -1706,6 +1799,39 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +[[package]] +name = "legion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bfd53bb4690a5ab2bd6d1c683461ee52763afe0d000929743708a256d9d9b1f" +dependencies = [ + "atomic_refcell", + "bit-set", + "downcast-rs", + "erased-serde", + "itertools", + "legion_codegen", + "parking_lot 0.11.2", + "paste", + "scoped-tls-hkt", + "serde", + "smallvec", + "thiserror", + "uuid", +] + +[[package]] +name = "legion_codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ad5ad7a361d7b2522010335d95fa73135cb3c6816bef22cc7a5d5861587ae1b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "thiserror", +] + [[package]] name = "libc" version = "0.2.124" @@ -2228,6 +2354,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + [[package]] name = "pbkdf2" version = "0.10.1" @@ -2706,7 +2838,7 @@ dependencies = [ "cbindgen", "cfg-if", "cmake", - "hotg-rune-core", + "hotg-rune-core 0.11.3", "hotg-rune-runtime", "libc", "log", @@ -2773,7 +2905,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd3904a4ba0a9d0211816177fd34b04c7095443f8cdacd11175064fe541c8fe2" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro2", "quote", "syn", @@ -2823,6 +2955,12 @@ dependencies = [ "syn", ] +[[package]] +name = "scoped-tls-hkt" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e9d7eaddb227e8fbaaa71136ae0e1e913ca159b86c7da82f3e8f0044ad3a63" + [[package]] name = "scoped_threadpool" version = "0.1.9" @@ -3087,7 +3225,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro-error", "proc-macro2", "quote", @@ -3109,7 +3247,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "339f799d8b549e3744c7ac7feb216383e4005d94bdb22561b3ab8f3b808ae9fb" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro2", "quote", "syn", @@ -3521,6 +3659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom", + "serde", ] [[package]] diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index 070ac984d5..8f462d61c0 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -186,13 +186,13 @@ mod tests { impl salsa::Database for Database {} - // The parsing process requires you to load proc-blocks and read files. You - // can satisfy these dependencies by implementing the corresponding traits. - impl FileSystem for Database { fn read(&self, path: &URI<'_>) -> Result, ReadError> { + // Note: The tests don't actually care about the value we get back. match path.scheme() { - Scheme::File => Ok(Vector::default()), + Scheme::HTTP | Scheme::HTTPS | Scheme::File => { + Ok(Vector::default()) + }, Scheme::Unregistered(s) if s.as_str() == "wapm" => { Ok(Vector::default()) }, diff --git a/crates/compiler/src/config.rs b/crates/compiler/src/config.rs index 94306e3ecb..6b9ab97537 100644 --- a/crates/compiler/src/config.rs +++ b/crates/compiler/src/config.rs @@ -2,9 +2,18 @@ use std::path::PathBuf; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct BuildConfig { + /// The directory all paths are resolved relative to. pub current_directory: PathBuf, + /// Unstable features which can enable extra options. + pub features: FeatureFlags, } +/// Flags used by the Rune compiler to enable experimental features. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub struct FeatureFlags {} + /// The build environment. #[salsa::query_group(EnvironmentStorage)] pub trait Environment { diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 4130d0d5eb..ecb21a4ac5 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -12,7 +12,7 @@ pub mod parse; pub mod type_check; pub use crate::{ - config::{BuildConfig, Environment, EnvironmentStorage}, + config::{BuildConfig, Environment, EnvironmentStorage, FeatureFlags}, filesystem::{FileSystem, ReadError}, im::Text, }; diff --git a/crates/rune-cli/Cargo.toml b/crates/rune-cli/Cargo.toml index f11d79da4a..699f39ef0d 100644 --- a/crates/rune-cli/Cargo.toml +++ b/crates/rune-cli/Cargo.toml @@ -24,8 +24,8 @@ codespan-reporting = "0.11.0" dirs = "4" dotenv = "0.15.0" env_logger = "0.9" -hotg-rune-compiler = { path = "../compiler", version = "^0.11.0"} -hotg-rune-core = { path = "../rune-core", version = "^0.11.0"} +hotg-rune-compiler = "^0.11.0" +hotg-rune-core = "^0.11.0" hotg-rune-proc-blocks = { version = "0.11.3", path = "../proc-blocks" } hotg-rune-runtime = { path = "../runtime", version = "^0.11.0", features = ["builtins", "wasm3", "wasmer"] } hotg-runecoral = "0.3.11" diff --git a/crates/rune-cli/src/build/mod.rs b/crates/rune-cli/src/build/mod.rs new file mode 100644 index 0000000000..a87fb4a979 --- /dev/null +++ b/crates/rune-cli/src/build/mod.rs @@ -0,0 +1,89 @@ +mod rune_v0; + +use std::path::PathBuf; + +use anyhow::{Context, Error}; +use codespan_reporting::term::termcolor::ColorChoice; +use once_cell::sync::Lazy; + +use crate::Unstable; + +#[derive(Debug, Clone, PartialEq, structopt::StructOpt)] +pub struct Build { + /// The Runefile to compile. + #[structopt(parse(from_os_str), default_value = "Runefile.yml")] + runefile: PathBuf, + /// Where to write the generated Rune. + #[structopt(short, long, parse(from_os_str))] + output: Option, + /// The directory to use when caching builds. + #[structopt(long, env)] + cache_dir: Option, + /// The directory that all paths are resolved relative to (Defaults to the + /// Runefile's directory) + #[structopt(short, long, env)] + current_dir: Option, + /// The name of the Rune (defaults to the Runefile directory's name). + #[structopt(short, long)] + name: Option, + /// Hide output from tools that rune may call. + #[structopt(short, long, conflicts_with = "verbose")] + quiet: bool, + /// Prints even more detailed information. + #[structopt(short, long, conflicts_with = "quiet")] + verbose: bool, + /// Compile the Rune without optimisations. + #[structopt(long)] + debug: bool, +} + +impl Build { + pub fn execute( + self, + color: ColorChoice, + unstable: Unstable, + ) -> Result<(), Error> { + rune_v0::execute(self, color, unstable) + } + + pub fn current_directory(&self) -> Result { + if let Some(dir) = &self.current_dir { + return Ok(dir.clone()); + } + + if let Some(parent) = + self.runefile.parent().and_then(|p| p.canonicalize().ok()) + { + return Ok(parent); + } + + std::env::current_dir() + .context("Unable to determine the current directory") + } + + pub fn name(&self) -> Result { + if let Some(name) = &self.name { + return Ok(name.clone()); + } + + let current_dir = self.current_directory()?; + + if let Some(name) = current_dir.file_name().and_then(|n| n.to_str()) { + return Ok(name.to_string()); + } + + Err(Error::msg("Unable to determine the Rune's name")) + } +} + +pub(crate) static DEFAULT_CACHE_DIR: Lazy = Lazy::new(|| { + let cache_dir = dirs::cache_dir() + .or_else(dirs::home_dir) + .unwrap_or_else(|| PathBuf::from(".")); + + cache_dir + .join("rune") + .join("runes") + .to_string_lossy() + .into_owned() +}); diff --git a/crates/rune-cli/src/build.rs b/crates/rune-cli/src/build/rune_v0.rs similarity index 51% rename from crates/rune-cli/src/build.rs rename to crates/rune-cli/src/build/rune_v0.rs index 69f6603ff9..c3fb7943a4 100644 --- a/crates/rune-cli/src/build.rs +++ b/crates/rune-cli/src/build/rune_v0.rs @@ -18,138 +18,64 @@ use hotg_rune_compiler::{ }, BuildContext, Verbosity, }; -use once_cell::sync::Lazy; -use crate::Unstable; - -#[derive(Debug, Clone, PartialEq, structopt::StructOpt)] -pub struct Build { - /// The Runefile to compile. - #[structopt(parse(from_os_str), default_value = "Runefile.yml")] - runefile: PathBuf, - /// Where to write the generated Rune. - #[structopt(short, long, parse(from_os_str))] - output: Option, - /// The directory to use when caching builds. - #[structopt(long, env)] - cache_dir: Option, - /// The directory that all paths are resolved relative to (Defaults to the - /// Runefile's directory) - #[structopt(short, long, env)] - current_dir: Option, - /// The name of the Rune (defaults to the Runefile directory's name). - #[structopt(short, long)] - name: Option, - /// Hide output from tools that rune may call. - #[structopt(short, long, conflicts_with = "verbose")] - quiet: bool, - /// Prints even more detailed information. - #[structopt(short, long, conflicts_with = "quiet")] - verbose: bool, - /// Compile the Rune without optimisations. - #[structopt(long)] - debug: bool, -} - -impl Build { - pub fn execute( - self, - color: ColorChoice, - unstable: Unstable, - ) -> Result<(), Error> { - let ctx = self.build_context()?; - let features = unstable.feature_flags(); - - log::debug!( - "Compiling {} in \"{}\"", - ctx.name, - ctx.working_directory.display() - ); - - let dest = self.output.unwrap_or_else(|| { - ctx.current_directory.join(&ctx.name).with_extension("rune") - }); - - let mut hooks = Hooks::new(dest, color, self.runefile); - hotg_rune_compiler::build_with_hooks(ctx, features, &mut hooks); - - match hooks.error { - None => Ok(()), - Some(e) => Err(e), - } - } - - fn build_context(&self) -> Result { - let verbosity = - Verbosity::from_quiet_and_verbose(self.quiet, self.verbose) - .context( - "The --verbose and --quiet flags can't be used together", - )?; - - let current_directory = self.current_directory()?; - let name = self.name()?; - - let working_directory = self - .cache_dir - .clone() - .unwrap_or_else(|| Path::new(&*DEFAULT_CACHE_DIR).join(&name)); - let runefile = - std::fs::read_to_string(&self.runefile).with_context(|| { - format!("Unable to read \"{}\"", self.runefile.display()) - })?; - - Ok(BuildContext { - name, - current_directory, - runefile, - verbosity, - working_directory, - optimized: !self.debug, - rune_version: Some(RuneVersion::new(env!("CARGO_PKG_VERSION"))), - }) - } - - fn current_directory(&self) -> Result { - if let Some(dir) = &self.current_dir { - return Ok(dir.clone()); - } - - if let Some(parent) = - self.runefile.parent().and_then(|p| p.canonicalize().ok()) - { - return Ok(parent); - } - - std::env::current_dir() - .context("Unable to determine the current directory") - } - - fn name(&self) -> Result { - if let Some(name) = &self.name { - return Ok(name.clone()); - } - - let current_dir = self.current_directory()?; - - if let Some(name) = current_dir.file_name().and_then(|n| n.to_str()) { - return Ok(name.to_string()); - } +use crate::{ + build::{Build, DEFAULT_CACHE_DIR}, + Unstable, +}; - Err(Error::msg("Unable to determine the Rune's name")) +pub(crate) fn execute( + build: Build, + color: ColorChoice, + unstable: Unstable, +) -> Result<(), Error> { + let ctx = build_context(&build)?; + let features = unstable.feature_flags(); + + log::debug!( + "Compiling {} in \"{}\"", + ctx.name, + ctx.working_directory.display() + ); + + let dest = build.output.unwrap_or_else(|| { + ctx.current_directory.join(&ctx.name).with_extension("rune") + }); + + let mut hooks = Hooks::new(dest, color, build.runefile); + hotg_rune_compiler::build_with_hooks(ctx, features, &mut hooks); + + match hooks.error { + None => Ok(()), + Some(e) => Err(e), } } -static DEFAULT_CACHE_DIR: Lazy = Lazy::new(|| { - let cache_dir = dirs::cache_dir() - .or_else(dirs::home_dir) - .unwrap_or_else(|| PathBuf::from(".")); - - cache_dir - .join("rune") - .join("runes") - .to_string_lossy() - .into_owned() -}); +fn build_context(b: &Build) -> Result { + let verbosity = Verbosity::from_quiet_and_verbose(b.quiet, b.verbose) + .context("The --verbose and --quiet flags can't be used together")?; + + let current_directory = b.current_directory()?; + let name = b.name()?; + + let working_directory = b + .cache_dir + .clone() + .unwrap_or_else(|| Path::new(&*DEFAULT_CACHE_DIR).join(&name)); + let runefile = std::fs::read_to_string(&b.runefile).with_context(|| { + format!("Unable to read \"{}\"", b.runefile.display()) + })?; + + Ok(BuildContext { + name, + current_directory, + runefile, + verbosity, + working_directory, + optimized: !b.debug, + rune_version: Some(RuneVersion::new(env!("CARGO_PKG_VERSION"))), + }) +} #[derive(Debug)] struct Hooks { diff --git a/crates/rune-cli/src/inspect/proc_block.rs b/crates/rune-cli/src/inspect/proc_block.rs index 37c8abce7b..3e8766c9d6 100644 --- a/crates/rune-cli/src/inspect/proc_block.rs +++ b/crates/rune-cli/src/inspect/proc_block.rs @@ -169,11 +169,6 @@ fn generate_project(dest: &PathBuf, filename: &Path) -> Result<(), Error> { let name = name.replace("-", "_"); write(dest.join("lib.rs"), LIB_RS_TEMPLATE.replace("$NAME", &name))?; - write( - dest.join("rust-toolchain.toml"), - hotg_rune_compiler::rust_toolchain().to_string(), - )?; - Ok(()) } diff --git a/crates/rune-cli/tests/integration.rs b/crates/rune-cli/tests/integration.rs index e545077941..2286cea7e3 100644 --- a/crates/rune-cli/tests/integration.rs +++ b/crates/rune-cli/tests/integration.rs @@ -19,7 +19,9 @@ fn project_root() -> PathBuf { ); } -fn example_dir() -> PathBuf { project_root().join("examples") } +fn example_dir() -> PathBuf { + project_root().join("examples") +} fn cache_dir() -> PathBuf { project_root().join("target").join(concat!( @@ -61,6 +63,10 @@ fn person_detection() { #[test] fn build_all_examples() { + // TODO: Enable these when all Rune examples have been migrated to the + // zipped format based on wit-files. + let exclude = ["sine"]; + let runefiles = WalkDir::new(example_dir()) .into_iter() .filter_map(|entry| entry.ok()) @@ -71,6 +77,11 @@ fn build_all_examples() { for runefile in runefiles { let path = runefile.path(); let name = path.parent().unwrap().file_name().unwrap(); + + if exclude.contains(&name.to_str().unwrap()) { + continue; + } + let cache_dir = cache_dir.join(name); let mut cmd = Command::cargo_bin("rune").unwrap(); From 49af8ab53345fd3f8b2a8031a98769b493d54d1e Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 10 May 2022 22:03:22 +0800 Subject: [PATCH 32/84] Added a switch for building with the new compiler --- crates/rune-cli/build.rs | 4 +++- crates/rune-cli/src/build/mod.rs | 29 ++++++++++++++++++++++++++-- crates/rune-cli/src/build/rune_v1.rs | 11 +++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 crates/rune-cli/src/build/rune_v1.rs diff --git a/crates/rune-cli/build.rs b/crates/rune-cli/build.rs index 28d52d3569..d36778f606 100644 --- a/crates/rune-cli/build.rs +++ b/crates/rune-cli/build.rs @@ -1 +1,3 @@ -fn main() { build_info_build::build_script(); } +fn main() { + build_info_build::build_script(); +} diff --git a/crates/rune-cli/src/build/mod.rs b/crates/rune-cli/src/build/mod.rs index a87fb4a979..e8e69de2dd 100644 --- a/crates/rune-cli/src/build/mod.rs +++ b/crates/rune-cli/src/build/mod.rs @@ -1,6 +1,7 @@ mod rune_v0; +mod rune_v1; -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; use anyhow::{Context, Error}; use codespan_reporting::term::termcolor::ColorChoice; @@ -35,6 +36,9 @@ pub struct Build { /// Compile the Rune without optimisations. #[structopt(long)] debug: bool, + /// The type of Rune to build. + #[structopt( long, short, env = "RUNE_TARGET", default_value = "abi-v0", parse(try_from_str), possible_values = &["abi-v0", "abi-v1"])] + target: Target, } impl Build { @@ -43,7 +47,10 @@ impl Build { color: ColorChoice, unstable: Unstable, ) -> Result<(), Error> { - rune_v0::execute(self, color, unstable) + match self.target { + Target::AbiV0 => rune_v0::execute(self, color, unstable), + Target::AbiV1 => rune_v1::execute(self, unstable), + } } pub fn current_directory(&self) -> Result { @@ -87,3 +94,21 @@ pub(crate) static DEFAULT_CACHE_DIR: Lazy = Lazy::new(|| { .to_string_lossy() .into_owned() }); + +#[derive(Debug, Clone, PartialEq)] +enum Target { + AbiV0, + AbiV1, +} + +impl FromStr for Target { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value { + "abi-v0" => Ok(Target::AbiV0), + "abi-v1" => Ok(Target::AbiV1), + _ => Err(Error::msg("Unknown ABI version")), + } + } +} diff --git a/crates/rune-cli/src/build/rune_v1.rs b/crates/rune-cli/src/build/rune_v1.rs new file mode 100644 index 0000000000..aebc8b4b66 --- /dev/null +++ b/crates/rune-cli/src/build/rune_v1.rs @@ -0,0 +1,11 @@ +use anyhow::Error; + +use crate::{Build, Unstable}; + +pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { + if !unstable.unstable { + anyhow::bail!("Building with the new ABI is still experimental. Please use the `--unstable` flag."); + } + + todo!() +} From 4bb14e14969afaf9956c8a1d7f0d6fe3a8c62be4 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 10 May 2022 22:06:16 +0800 Subject: [PATCH 33/84] Renamed the target modules to something more appropriate --- crates/rune-cli/src/build/{rune_v0.rs => abi_v0.rs} | 0 crates/rune-cli/src/build/{rune_v1.rs => abi_v1.rs} | 0 crates/rune-cli/src/build/mod.rs | 8 ++++---- 3 files changed, 4 insertions(+), 4 deletions(-) rename crates/rune-cli/src/build/{rune_v0.rs => abi_v0.rs} (100%) rename crates/rune-cli/src/build/{rune_v1.rs => abi_v1.rs} (100%) diff --git a/crates/rune-cli/src/build/rune_v0.rs b/crates/rune-cli/src/build/abi_v0.rs similarity index 100% rename from crates/rune-cli/src/build/rune_v0.rs rename to crates/rune-cli/src/build/abi_v0.rs diff --git a/crates/rune-cli/src/build/rune_v1.rs b/crates/rune-cli/src/build/abi_v1.rs similarity index 100% rename from crates/rune-cli/src/build/rune_v1.rs rename to crates/rune-cli/src/build/abi_v1.rs diff --git a/crates/rune-cli/src/build/mod.rs b/crates/rune-cli/src/build/mod.rs index e8e69de2dd..82e037dde1 100644 --- a/crates/rune-cli/src/build/mod.rs +++ b/crates/rune-cli/src/build/mod.rs @@ -1,5 +1,5 @@ -mod rune_v0; -mod rune_v1; +mod abi_v0; +mod abi_v1; use std::{path::PathBuf, str::FromStr}; @@ -48,8 +48,8 @@ impl Build { unstable: Unstable, ) -> Result<(), Error> { match self.target { - Target::AbiV0 => rune_v0::execute(self, color, unstable), - Target::AbiV1 => rune_v1::execute(self, unstable), + Target::AbiV0 => abi_v0::execute(self, color, unstable), + Target::AbiV1 => abi_v1::execute(self, unstable), } } From 70f39ee39138677129638bb2d44dc0af9038b59f Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 10 May 2022 23:25:49 +0800 Subject: [PATCH 34/84] Wire up "tracing" as the logger --- Cargo.lock | 7 ++- crates/compiler/src/codegen/query.rs | 3 +- crates/compiler/src/config.rs | 9 ++- crates/compiler/src/im.rs | 6 ++ crates/rune-cli/Cargo.toml | 7 ++- crates/rune-cli/src/bin/rune.rs | 28 ++++++--- crates/rune-cli/src/build/abi_v0.rs | 4 +- crates/rune-cli/src/build/abi_v1.rs | 76 ++++++++++++++++++++++- crates/rune-cli/src/inspect/proc_block.rs | 8 +-- crates/rune-cli/src/inspect/rune.rs | 4 +- crates/rune-cli/src/lib.rs | 11 ---- crates/rune-cli/src/run.rs | 6 +- 12 files changed, 129 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e766da64e7..74d3333ac6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1325,7 +1325,7 @@ dependencies = [ "criterion", "dirs", "dotenv", - "env_logger", + "hotg-rune-compiler 0.11.3", "hotg-rune-compiler 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", "hotg-rune-core 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", "hotg-rune-proc-blocks 0.11.3", @@ -1335,17 +1335,20 @@ dependencies = [ "human-panic", "image", "indexmap", - "log", "once_cell", "predicates", "rand 0.8.5", "regex", + "salsa", "serde", "serde_json", "structopt", "strum", "tempdir", "tempfile", + "tracing", + "tracing-subscriber", + "uriparse", "walkdir", "wasmparser 0.81.0", ] diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index 8f462d61c0..a22264ed76 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -175,7 +175,7 @@ mod tests { use super::*; use crate::{ parse::Frontend, parse::FrontendStorage, BuildConfig, Environment, - EnvironmentStorage, FileSystem, ReadError, + EnvironmentStorage, FeatureFlags, FileSystem, ReadError, }; #[derive(Default)] @@ -216,6 +216,7 @@ mod tests { let mut db = Database::default(); db.set_config(BuildConfig { current_directory: sine_dir.clone(), + features: FeatureFlags::stable(), }); db.set_src(runefile.into()); diff --git a/crates/compiler/src/config.rs b/crates/compiler/src/config.rs index 6b9ab97537..9cacfdfaef 100644 --- a/crates/compiler/src/config.rs +++ b/crates/compiler/src/config.rs @@ -8,12 +8,19 @@ pub struct BuildConfig { pub features: FeatureFlags, } -/// Flags used by the Rune compiler to enable experimental features. +/// Flags used by the Rune compiler to enable features. #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, )] pub struct FeatureFlags {} +impl FeatureFlags { + /// Enable all stable features. + pub fn stable() -> Self { + FeatureFlags {} + } +} + /// The build environment. #[salsa::query_group(EnvironmentStorage)] pub trait Environment { diff --git a/crates/compiler/src/im.rs b/crates/compiler/src/im.rs index c9f8dbb18b..9116b9537b 100644 --- a/crates/compiler/src/im.rs +++ b/crates/compiler/src/im.rs @@ -154,6 +154,12 @@ impl FromIterator for Vector { } } +impl AsRef<[A]> for Vector { + fn as_ref(&self) -> &[A] { + self.0.as_ref() + } +} + #[derive( Debug, PartialEq, diff --git a/crates/rune-cli/Cargo.toml b/crates/rune-cli/Cargo.toml index 699f39ef0d..7dff90ef5b 100644 --- a/crates/rune-cli/Cargo.toml +++ b/crates/rune-cli/Cargo.toml @@ -23,7 +23,6 @@ chrono = { version = "0.4.19", features = ["std"] } codespan-reporting = "0.11.0" dirs = "4" dotenv = "0.15.0" -env_logger = "0.9" hotg-rune-compiler = "^0.11.0" hotg-rune-core = "^0.11.0" hotg-rune-proc-blocks = { version = "0.11.3", path = "../proc-blocks" } @@ -33,14 +32,18 @@ hound = "3.4.0" human-panic = "1.0.3" image = "0.23.14" indexmap = "1.6.2" -log = "0.4.11" once_cell = "1.7.0" +query_based_compiler = { version = "0.11.3", path = "../compiler", package = "hotg-rune-compiler" } rand = "0.8.3" regex = "1.5.4" +salsa = "0.16.1" serde = { version = "1.0.125", features = ["derive"] } serde_json = "1.0.64" structopt = "0.3.21" strum = { version = "0.22.0", features = ["derive"] } +tracing = { version = "0.1.34", features = ["log"] } +tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } +uriparse = "0.6.4" wasmparser = "0.81" [dev-dependencies] diff --git a/crates/rune-cli/src/bin/rune.rs b/crates/rune-cli/src/bin/rune.rs index cc1b793ab9..a2e26bb5c4 100644 --- a/crates/rune-cli/src/bin/rune.rs +++ b/crates/rune-cli/src/bin/rune.rs @@ -1,17 +1,31 @@ use anyhow::Error; -use env_logger::Env; use hotg_rune_cli::{ Build, ColorChoice, Format, Graph, Inspect, ModelInfo, Run, Unstable, Version, }; -use log::LevelFilter; use structopt::{clap::AppSettings, StructOpt}; use strum::VariantNames; +use tracing_subscriber::EnvFilter; fn main() -> Result<(), Error> { let _ = dotenv::dotenv(); human_panic::setup_panic!(); + if std::env::var_os("RUST_LOG").is_none() { + // Some modules are known to generate loads of logs that aren't relevant + // so we only log warnings by default. + std::env::set_var( + "RUST_LOG", + [ + "info", + "cranelift_codegen=warn", + "regalloc=warn", + "salsa=warn", + ] + .join(","), + ); + } + let Args { colour, cmd, @@ -19,14 +33,8 @@ fn main() -> Result<(), Error> { unstable, } = Args::from_args(); - let env = Env::default().default_filter_or("warn"); - env_logger::Builder::from_env(env) - .format_timestamp_millis() - .format_indent(Some(2)) - .write_style(colour.into()) - // Some modules are known to generate loads of logs that aren't relevant - .filter_module("cranelift_codegen", LevelFilter::Warn) - .filter_module("regalloc", LevelFilter::Warn) + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) .init(); match cmd { diff --git a/crates/rune-cli/src/build/abi_v0.rs b/crates/rune-cli/src/build/abi_v0.rs index c3fb7943a4..c9767696da 100644 --- a/crates/rune-cli/src/build/abi_v0.rs +++ b/crates/rune-cli/src/build/abi_v0.rs @@ -32,7 +32,7 @@ pub(crate) fn execute( let ctx = build_context(&build)?; let features = unstable.feature_flags(); - log::debug!( + tracing::debug!( "Compiling {} in \"{}\"", ctx.name, ctx.working_directory.display() @@ -109,7 +109,7 @@ impl Hooks { format!("Unable to write to \"{}\"", self.dest.display()) })?; - log::info!("The Rune was written to \"{}\"", self.dest.display()); + tracing::info!("The Rune was written to \"{}\"", self.dest.display()); Ok(()) } diff --git a/crates/rune-cli/src/build/abi_v1.rs b/crates/rune-cli/src/build/abi_v1.rs index aebc8b4b66..e4d6aa215d 100644 --- a/crates/rune-cli/src/build/abi_v1.rs +++ b/crates/rune-cli/src/build/abi_v1.rs @@ -1,4 +1,14 @@ -use anyhow::Error; +use std::{path::PathBuf, sync::Arc}; + +use anyhow::{Context, Error}; +use query_based_compiler::{ + codegen::{Codegen, CodegenStorage}, + im::Vector, + parse::{Frontend, FrontendStorage}, + EnvironmentStorage, FileSystem, ReadError, +}; +use salsa::Storage; +use uriparse::{Scheme, URI}; use crate::{Build, Unstable}; @@ -7,5 +17,67 @@ pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { anyhow::bail!("Building with the new ABI is still experimental. Please use the `--unstable` flag."); } - todo!() + let runefile = + std::fs::read_to_string(&build.runefile).with_context(|| { + format!("Unable to read \"{}\"", build.runefile.display()) + })?; + let name = build.name()?; + + let mut db = Database { + storage: Storage::default(), + current_dir: build.current_directory()?, + }; + + db.set_src(runefile.into()); + let archive = db.rune_archive()?; + + let dest = build + .output + .unwrap_or_else(|| db.current_dir.join(&name).with_extension("rune")); + + tracing::info!(path = %dest.display(), "Saving the compiled Rune"); + + std::fs::write(&dest, &archive) + .with_context(|| format!("Unable to save to \"{}\"", dest.display()))?; + + Ok(()) +} + +#[salsa::database(CodegenStorage, EnvironmentStorage, FrontendStorage)] +struct Database { + storage: Storage, + current_dir: PathBuf, +} + +impl salsa::Database for Database {} + +impl FileSystem for Database { + fn read(&self, path: &URI<'_>) -> Result, ReadError> { + match path.scheme() { + Scheme::FileSystem => read_file(path.path()), + Scheme::Unregistered(u) if u.as_str().is_empty() => { + read_file(path.path()) + }, + + other => Err(ReadError::UnsupportedScheme { + scheme: other.as_str().into(), + }), + } + } +} + +fn read_file(path: &uriparse::Path<'_>) -> Result, ReadError> { + let mut full_path = PathBuf::new(); + + if path.is_absolute() { + full_path.push(std::path::Component::RootDir); + } + + for segment in path.segments() { + full_path.push(segment.as_str()); + } + + std::fs::read(&full_path) + .map(Vector::from) + .map_err(|e| ReadError::Other(Arc::new(e) as Arc<_>)) } diff --git a/crates/rune-cli/src/inspect/proc_block.rs b/crates/rune-cli/src/inspect/proc_block.rs index 3e8766c9d6..f2ff2952c6 100644 --- a/crates/rune-cli/src/inspect/proc_block.rs +++ b/crates/rune-cli/src/inspect/proc_block.rs @@ -12,11 +12,11 @@ use hotg_rune_proc_blocks::{ use crate::{inspect::wasm_custom_sections, Format}; pub fn inspect(format: Format, proc_block_dir: &Path) -> Result<(), Error> { - log::info!("Inspecting \"{}\"", proc_block_dir.display()); + tracing::info!("Inspecting \"{}\"", proc_block_dir.display()); let dest = cache_dir(proc_block_dir); - log::debug!("Writing probe to \"{}\"", dest.display()); + tracing::debug!("Writing probe to \"{}\"", dest.display()); generate_project(&dest, proc_block_dir) .context("Unable to generate the probe project")?; @@ -27,7 +27,7 @@ pub fn inspect(format: Format, proc_block_dir: &Path) -> Result<(), Error> { .arg("wasm32-unknown-unknown") .current_dir(&dest); - log::debug!("Executing {:?}", cmd); + tracing::debug!("Executing {:?}", cmd); let status = cmd .status() @@ -45,7 +45,7 @@ pub fn inspect(format: Format, proc_block_dir: &Path) -> Result<(), Error> { let wasm = std::fs::read(&binary) .with_context(|| format!("Unable to read \"{}\"", binary.display()))?; - log::debug!("Read {} bytes from \"{}\"", wasm.len(), binary.display()); + tracing::debug!("Read {} bytes from \"{}\"", wasm.len(), binary.display()); let sections = wasm_custom_sections(&wasm) .context("Unable to parse the WebAssembly module")?; diff --git a/crates/rune-cli/src/inspect/rune.rs b/crates/rune-cli/src/inspect/rune.rs index 6ee4a7e78a..0e15f52113 100644 --- a/crates/rune-cli/src/inspect/rune.rs +++ b/crates/rune-cli/src/inspect/rune.rs @@ -193,7 +193,7 @@ impl Metadata { meta.rune = Some(rune); }, Err(e) => { - log::warn!( + tracing::warn!( "Unable to deserialize the Rune graph: {}", e ); @@ -206,7 +206,7 @@ impl Metadata { meta.version = Some(v); }, Err(e) => { - log::warn!( + tracing::warn!( "Unable to deserialize the version: {}", e ); diff --git a/crates/rune-cli/src/lib.rs b/crates/rune-cli/src/lib.rs index ebcc9939d6..1a8c0bb337 100644 --- a/crates/rune-cli/src/lib.rs +++ b/crates/rune-cli/src/lib.rs @@ -7,7 +7,6 @@ mod unstable; mod version; use codespan_reporting::term::termcolor; -use env_logger::WriteStyle; pub use crate::{ build::Build, graph::Graph, inspect::Inspect, model_info::ModelInfo, @@ -34,16 +33,6 @@ impl From for termcolor::ColorChoice { } } -impl From for WriteStyle { - fn from(c: ColorChoice) -> WriteStyle { - match c { - ColorChoice::Always => WriteStyle::Always, - ColorChoice::Auto => WriteStyle::Auto, - ColorChoice::Never => WriteStyle::Never, - } - } -} - #[derive( Debug, Copy, Clone, PartialEq, strum::EnumVariantNames, strum::EnumString, )] diff --git a/crates/rune-cli/src/run.rs b/crates/rune-cli/src/run.rs index 63a34e27ec..be547ee18e 100644 --- a/crates/rune-cli/src/run.rs +++ b/crates/rune-cli/src/run.rs @@ -70,7 +70,7 @@ pub struct Run { impl Run { pub fn execute(self) -> Result<(), Error> { - log::info!("Running rune: {}", self.rune.display()); + tracing::info!("Running rune: {}", self.rune.display()); let rune = std::fs::read(&self.rune).with_context(|| { format!("Unable to read \"{}\"", self.rune.display()) @@ -83,7 +83,7 @@ impl Run { self.load_resources(runtime.resources())?; let caps = runtime.capabilities().clone(); - log::debug!("Loading capabilities {:?}", caps); + tracing::debug!("Loading capabilities {:?}", caps); runtime.input_tensors().extend(self.load_inputs(caps)?); runtime.predict().context("Prediction failed")?; @@ -104,7 +104,7 @@ impl Run { let mut inputs = HashMap::new(); for (id, metadata) in caps { - log::debug!("Loading {:?}", metadata); + tracing::debug!("Loading {:?}", metadata); let NodeMetadata { kind, arguments, .. } = metadata; From 47997e09455068fbb90acbb4d15d0a534a152f38 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 13 May 2022 00:44:32 +0800 Subject: [PATCH 35/84] Fixed up the sine example --- crates/compiler/examples/compile.rs | 4 +++- crates/compiler/src/codegen/query.rs | 36 +++++++++++++++++++++++++--- examples/sine/Runefile.yml | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/crates/compiler/examples/compile.rs b/crates/compiler/examples/compile.rs index 94551aa042..2979126d99 100644 --- a/crates/compiler/examples/compile.rs +++ b/crates/compiler/examples/compile.rs @@ -4,7 +4,8 @@ use hotg_rune_compiler::{ codegen::{Codegen, CodegenStorage}, im::Vector, parse::{Frontend, FrontendStorage}, - BuildConfig, Environment, EnvironmentStorage, FileSystem, ReadError, + BuildConfig, Environment, EnvironmentStorage, FeatureFlags, FileSystem, + ReadError, }; use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; use uriparse::{Scheme, URI}; @@ -31,6 +32,7 @@ fn main() { db.set_src(src.into()); db.set_config(BuildConfig { current_directory: parent.clone(), + features: FeatureFlags::stable(), }); let archive = db.rune_archive().unwrap(); diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index a22264ed76..d5f083e564 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -210,12 +210,42 @@ mod tests { .parent() .unwrap(); let sine_dir = project_root.join("examples").join("sine"); - let runefile = - std::fs::read_to_string(sine_dir.join("Runefile.yml")).unwrap(); + let runefile = r#" + version: 1 + image: runicos/base + pipeline: + rand: + capability: RAW + args: + length: 4 + outputs: + - type: F32 + dimensions: [1, 1] + mod360: + proc-block: "https://func.hotg.ai/function/sbfs/pb/argmax.wasm" + inputs: + - rand + outputs: + - type: F32 + dimensions: [1, 1] + args: + modulus: 360.0 + sine: + model: "./sinemodel.tflite" + inputs: + - mod360 + outputs: + - type: F32 + dimensions: [1, 1] + serial: + out: serial + inputs: + - sine + "#; let mut db = Database::default(); db.set_config(BuildConfig { - current_directory: sine_dir.clone(), + current_directory: sine_dir, features: FeatureFlags::stable(), }); db.set_src(runefile.into()); diff --git a/examples/sine/Runefile.yml b/examples/sine/Runefile.yml index e6ae56628f..ff1f988c62 100644 --- a/examples/sine/Runefile.yml +++ b/examples/sine/Runefile.yml @@ -11,7 +11,7 @@ pipeline: - 1 - 1 mod360: - proc-block: "https://func.hotg.ai/function/sbfs/pb/argmax.wasm" + proc-block: "hotg-ai/proc-blocks@v0.11.3#modulo" inputs: - rand outputs: From f71c3d2e500d3212689ee59d61ae42fc48034371 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 13 May 2022 16:34:44 +0800 Subject: [PATCH 36/84] We forgot to actually build the sine rune that our C API test requires --- bindings/native/tests/c_api.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bindings/native/tests/c_api.rs b/bindings/native/tests/c_api.rs index 836b3e62d2..cf7bd900af 100644 --- a/bindings/native/tests/c_api.rs +++ b/bindings/native/tests/c_api.rs @@ -12,10 +12,14 @@ static SINE_RUNE: Lazy> = Lazy::new(|| { let sine_dir = workspace_root.join("examples").join("sine"); let runefile = sine_dir.join("Runefile.yml"); - Command::new(env!("CARGO")) + let status = Command::new(env!("CARGO")) .arg("rune") .arg("build") - .arg(&runefile); + .arg(&runefile) + .status() + .expect("Unable to run cargo"); + + assert!(status.success(), "Building the Rune failed"); let rune = sine_dir.join("sine.rune"); From 7f7fe55ccf2d0f79e83ab40a238c9323e2a87ba0 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 13 May 2022 16:34:53 +0800 Subject: [PATCH 37/84] Cargo fmt and logging cleanups --- bindings/native/src/error.rs | 8 ++++++-- bindings/native/src/input_tensors.rs | 8 ++++++-- bindings/native/src/metadata.rs | 4 +++- bindings/native/src/runtime.rs | 8 ++++++-- bindings/native/tests/getting_started.rs | 8 ++++++-- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/bindings/native/src/error.rs b/bindings/native/src/error.rs index ecb058b0a7..52913de2bd 100644 --- a/bindings/native/src/error.rs +++ b/bindings/native/src/error.rs @@ -65,13 +65,17 @@ impl Error { } impl From for Error { - fn from(e: anyhow::Error) -> Error { Error(e) } + fn from(e: anyhow::Error) -> Error { + Error(e) + } } impl Deref for Error { type Target = anyhow::Error; - fn deref(&self) -> &Self::Target { &self.0 } + fn deref(&self) -> &Self::Target { + &self.0 + } } /// Create a new `Error` with the provided error message. diff --git a/bindings/native/src/input_tensors.rs b/bindings/native/src/input_tensors.rs index ef324ceadd..34a973044d 100644 --- a/bindings/native/src/input_tensors.rs +++ b/bindings/native/src/input_tensors.rs @@ -20,11 +20,15 @@ pub struct InputTensors(NonNull>); impl std::ops::Deref for InputTensors { type Target = HashMap; - fn deref(&self) -> &Self::Target { unsafe { self.0.as_ref() } } + fn deref(&self) -> &Self::Target { + unsafe { self.0.as_ref() } + } } impl std::ops::DerefMut for InputTensors { - fn deref_mut(&mut self) -> &mut Self::Target { unsafe { self.0.as_mut() } } + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { self.0.as_mut() } + } } impl From<&'_ mut HashMap> for InputTensors { diff --git a/bindings/native/src/metadata.rs b/bindings/native/src/metadata.rs index 07b5575a65..68c8d3e5b9 100644 --- a/bindings/native/src/metadata.rs +++ b/bindings/native/src/metadata.rs @@ -13,7 +13,9 @@ pub struct Metadata(Vec); impl std::ops::Deref for Metadata { type Target = [Node]; - fn deref(&self) -> &Self::Target { &self.0 } + fn deref(&self) -> &Self::Target { + &self.0 + } } impl From<&'_ HashMap> for Metadata { diff --git a/bindings/native/src/runtime.rs b/bindings/native/src/runtime.rs index a4222714fc..81d4032d9b 100644 --- a/bindings/native/src/runtime.rs +++ b/bindings/native/src/runtime.rs @@ -18,11 +18,15 @@ pub struct Runtime { impl Deref for Runtime { type Target = RustRuntime; - fn deref(&self) -> &Self::Target { &self.inner } + fn deref(&self) -> &Self::Target { + &self.inner + } } impl DerefMut for Runtime { - fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } } /// Data used when loading a Rune. diff --git a/bindings/native/tests/getting_started.rs b/bindings/native/tests/getting_started.rs index e296e1cd3f..5983997aa6 100644 --- a/bindings/native/tests/getting_started.rs +++ b/bindings/native/tests/getting_started.rs @@ -42,6 +42,8 @@ fn main() -> Result<(), Error> { .context("Unable to generate the project")?; run_build_script(&ctx).context("Unable to run the build script")?; + tracing::info!("Integration test passed ✅"); + Ok(()) } @@ -148,7 +150,7 @@ struct Context { } impl Context { - #[tracing::instrument("loading_context")] + #[tracing::instrument("loading_native_context")] fn from_env() -> Result { let crate_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let workspace_root = crate_dir @@ -174,7 +176,9 @@ impl Context { }) } - fn build_script(&self) -> PathBuf { self.build_dir.join("build.sh") } + fn build_script(&self) -> PathBuf { + self.build_dir.join("build.sh") + } } #[tracing::instrument(skip(ctx, parsed))] From 85a7e6df06d49ef65ad9a5b1a91bfcee48b9a9ab Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 17 May 2022 22:31:22 +0800 Subject: [PATCH 38/84] Forgot to set the build config --- crates/rune-cli/src/build/abi_v1.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/rune-cli/src/build/abi_v1.rs b/crates/rune-cli/src/build/abi_v1.rs index e4d6aa215d..5aa6b59e7d 100644 --- a/crates/rune-cli/src/build/abi_v1.rs +++ b/crates/rune-cli/src/build/abi_v1.rs @@ -5,7 +5,8 @@ use query_based_compiler::{ codegen::{Codegen, CodegenStorage}, im::Vector, parse::{Frontend, FrontendStorage}, - EnvironmentStorage, FileSystem, ReadError, + BuildConfig, Environment, EnvironmentStorage, FeatureFlags, FileSystem, + ReadError, }; use salsa::Storage; use uriparse::{Scheme, URI}; @@ -28,6 +29,10 @@ pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { current_dir: build.current_directory()?, }; + db.set_config(BuildConfig { + current_directory: db.current_dir.clone(), + features: FeatureFlags::stable(), + }); db.set_src(runefile.into()); let archive = db.rune_archive()?; From 829430650d470e50c9d98c8b3850a1355ae81362 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 19 May 2022 19:05:00 +0800 Subject: [PATCH 39/84] Wired up WAPM URIs --- Cargo.lock | 147 ++++++++++++++++++---- crates/compiler/src/filesystem.rs | 27 +++- crates/compiler/src/parse/query.rs | 17 ++- crates/compiler/src/parse/yaml.rs | 45 ++++++- crates/rune-cli/Cargo.toml | 2 + crates/rune-cli/benches/rune_benchmark.rs | 4 +- crates/rune-cli/src/build/abi_v1.rs | 65 +++++++++- examples/sine/Runefile.abi-v1.yml | 36 ++++++ 8 files changed, 307 insertions(+), 36 deletions(-) create mode 100644 examples/sine/Runefile.abi-v1.yml diff --git a/Cargo.lock b/Cargo.lock index 74d3333ac6..1ad883655d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,6 +47,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "aho-corasick" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" +dependencies = [ + "memchr", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -161,7 +170,7 @@ dependencies = [ "peeking_take_while", "proc-macro2", "quote", - "regex", + "regex 1.5.5", "rustc-hash", "shlex 1.1.0", "which", @@ -665,7 +674,7 @@ dependencies = [ "oorandom", "plotters", "rayon", - "regex", + "regex 1.5.5", "serde", "serde_cbor", "serde_derive", @@ -987,7 +996,7 @@ dependencies = [ "atty", "humantime", "log", - "regex", + "regex 1.5.5", "termcolor", ] @@ -1013,7 +1022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d95b4efe5be9104a4a18a9916e86654319895138be727b229820c39257c30dda" dependencies = [ "bit-set", - "regex", + "regex 1.5.5", ] [[package]] @@ -1074,7 +1083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ "matches", - "percent-encoding", + "percent-encoding 2.1.0", ] [[package]] @@ -1214,7 +1223,7 @@ dependencies = [ "libc", "libgit2-sys", "log", - "url", + "url 2.2.2", ] [[package]] @@ -1229,11 +1238,11 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.18", "bstr", "fnv", "log", - "regex", + "regex 1.5.5", ] [[package]] @@ -1337,8 +1346,10 @@ dependencies = [ "indexmap", "once_cell", "predicates", + "queryst", "rand 0.8.5", - "regex", + "regex 1.5.5", + "reqwest", "salsa", "serde", "serde_json", @@ -1362,7 +1373,7 @@ dependencies = [ "jsonschema", "once_cell", "pretty_assertions 1.2.1", - "regex", + "regex 1.5.5", "reqwest", "salsa", "schemars", @@ -1397,7 +1408,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "regex", + "regex 1.5.5", "schemars", "serde", "serde_json", @@ -1433,7 +1444,7 @@ dependencies = [ "log", "once_cell", "rayon", - "regex", + "regex 1.5.5", "structopt", "walkdir", ] @@ -1622,6 +1633,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.2.3" @@ -1773,14 +1795,14 @@ dependencies = [ "memchr", "num-cmp", "parking_lot 0.12.0", - "percent-encoding", - "regex", + "percent-encoding 2.1.0", + "regex 1.5.5", "reqwest", "serde", "serde_json", "structopt", "time 0.3.9", - "url", + "url 2.2.2", "uuid", ] @@ -2286,7 +2308,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3df761f6470298359f84fcfb60d86db02acc22c251c37265c07a3d1057d2389" dependencies = [ - "regex", + "regex 1.5.5", ] [[package]] @@ -2381,6 +2403,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -2471,7 +2499,7 @@ dependencies = [ "itertools", "normalize-line-endings", "predicates-core", - "regex", + "regex 1.5.5", ] [[package]] @@ -2585,6 +2613,19 @@ dependencies = [ "unicase", ] +[[package]] +name = "queryst" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e29c87f17d2fe0e4a83dd245c27b7b2d7db2232da356b4ceeb9b96ff75cd902" +dependencies = [ + "lazy_static", + "regex 0.2.11", + "serde", + "serde_json", + "url 1.7.2", +] + [[package]] name = "quote" version = "1.0.18" @@ -2716,15 +2757,28 @@ dependencies = [ "smallvec", ] +[[package]] +name = "regex" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" +dependencies = [ + "aho-corasick 0.6.10", + "memchr", + "regex-syntax 0.5.6", + "thread_local 0.3.6", + "utf8-ranges", +] + [[package]] name = "regex" version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.18", "memchr", - "regex-syntax", + "regex-syntax 0.6.25", ] [[package]] @@ -2733,7 +2787,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.25", +] + +[[package]] +name = "regex-syntax" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" +dependencies = [ + "ucd-util", ] [[package]] @@ -2794,14 +2857,14 @@ dependencies = [ "log", "mime", "native-tls", - "percent-encoding", + "percent-encoding 2.1.0", "pin-project-lite", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", - "url", + "url 2.2.2", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -3357,6 +3420,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +dependencies = [ + "lazy_static", +] + [[package]] name = "thread_local" version = "1.1.4" @@ -3541,10 +3613,10 @@ dependencies = [ "ansi_term", "lazy_static", "matchers", - "regex", + "regex 1.5.5", "sharded-slab", "smallvec", - "thread_local", + "thread_local 1.1.4", "tracing", "tracing-core", "tracing-log", @@ -3591,6 +3663,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" +[[package]] +name = "ucd-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85f514e095d348c279b1e5cd76795082cf15bd59b93207832abe0b1d8fed236" + [[package]] name = "unicase" version = "2.6.0" @@ -3643,6 +3721,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + [[package]] name = "url" version = "2.2.2" @@ -3650,11 +3739,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", - "idna", + "idna 0.2.3", "matches", - "percent-encoding", + "percent-encoding 2.1.0", ] +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "uuid" version = "0.8.2" diff --git a/crates/compiler/src/filesystem.rs b/crates/compiler/src/filesystem.rs index f01bfc641e..d10468ea32 100644 --- a/crates/compiler/src/filesystem.rs +++ b/crates/compiler/src/filesystem.rs @@ -1,4 +1,7 @@ -use std::sync::Arc; +use std::{ + fmt::{self, Debug, Display, Formatter}, + sync::Arc, +}; use uriparse::URI; @@ -20,6 +23,28 @@ pub enum ReadError { } impl ReadError { + pub fn msg(error_message: impl Display + Send + Sync + 'static) -> Self { + struct Message(T); + + impl Display for Message { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } + } + + impl Debug for Message { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_tuple("Message") + .field(&format_args!("{}", self.0)) + .finish() + } + } + + impl std::error::Error for Message {} + + ReadError::other(Message(error_message)) + } + pub fn other( error: impl std::error::Error + Send + Sync + 'static, ) -> Self { diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index ec35075021..052d21e114 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -6,7 +6,8 @@ use crate::{ im::{OrdMap, Vector}, parse::{ Document, DocumentV1, ItemType, ModelStage, NotFound, ParseFailed, - Path, ProcBlockStage, ResourceDeclaration, Stage, WrongItemType, + Path, ProcBlockStage, ResourceDeclaration, Stage, WellKnownPath, + WrongItemType, }, BuildConfig, Environment, FileSystem, Text, }; @@ -114,7 +115,7 @@ fn parse_runefile( .map_err(|e| ParseFailed { error: Arc::new(e) }) } -#[tracing::instrument(level = "debug", skip(db))] +#[tracing::instrument(level = "debug", skip(db), err)] fn parse(db: &dyn Frontend) -> Result, crate::Error> { db.parse_runefile(db.src()) .map(|d| Arc::new(Document::clone(&d).to_v1())) @@ -263,6 +264,7 @@ fn read(db: &dyn Frontend, path: &Path) -> Result, crate::Error> { fn file_uri(db: &dyn Frontend, path: &Path) -> Result, URIError> { match path { + Path::WellKnown(w) => Ok(wapm_uri(*w)), Path::Uri(u) => Ok(u.to_owned()), Path::FileSystem(path) => { let BuildConfig { @@ -282,3 +284,14 @@ fn file_uri(db: &dyn Frontend, path: &Path) -> Result, URIError> { }, } } + +fn wapm_uri(w: WellKnownPath) -> URI<'static> { + let uri = match w { + WellKnownPath::Accel => "wapm:///hotg-ai/accelerometer-input", + WellKnownPath::Image => "wapm:///hotg-ai/image-input", + WellKnownPath::Raw => "wapm:///hotg-ai/tensor-input", + WellKnownPath::Sound => "wapm:///hotg-ai/sound-input", + }; + + uri.try_into().expect("Should never fail") +} diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index fb902e54c7..10dd257992 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -228,6 +228,7 @@ pub struct ProcBlockStage { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum Path { + WellKnown(WellKnownPath), Uri(URI<'static>), FileSystem(String), } @@ -256,6 +257,10 @@ impl FromStr for Path { type Err = URIError; fn from_str(s: &str) -> Result { + if let Ok(well_known) = s.parse() { + return Ok(Path::WellKnown(well_known)); + } + match URI::try_from(s) { Ok(u) => Ok(Path::Uri(u.into_owned())), Err(URIError::NotURI) => Ok(Path::FileSystem(s.to_string())), @@ -267,6 +272,7 @@ impl FromStr for Path { impl Display for Path { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { + Path::WellKnown(w) => w.fmt(f), Path::Uri(u) => u.fmt(f), Path::FileSystem(p) => p.fmt(f), } @@ -283,6 +289,39 @@ impl JsonSchema for Path { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum WellKnownPath { + Accel, + Image, + Raw, + Sound, +} + +impl Display for WellKnownPath { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + WellKnownPath::Accel => "ACCEL".fmt(f), + WellKnownPath::Image => "IMAGE".fmt(f), + WellKnownPath::Raw => "RAW".fmt(f), + WellKnownPath::Sound => "SOUND".fmt(f), + } + } +} + +impl FromStr for WellKnownPath { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "ACCEL" | "accel" => Ok(WellKnownPath::Accel), + "IMAGE" | "image" => Ok(WellKnownPath::Image), + "RAW" | "raw" => Ok(WellKnownPath::Raw), + "SOUND" | "sound" => Ok(WellKnownPath::Sound), + _ => Err(()), + } + } +} + /// A stage which reads inputs from the runtime. #[derive( Debug, @@ -296,7 +335,7 @@ impl JsonSchema for Path { pub struct CapabilityStage { /// What type of capability to use ("IMAGE", "SOUND", etc.). #[schemars(required)] - pub capability: String, + pub capability: Path, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub outputs: Vec, #[serde(default, skip_serializing_if = "IndexMap::is_empty")] @@ -1073,7 +1112,7 @@ pipeline: image: "runicos/base".parse().unwrap(), pipeline: map! { audio: Stage::Capability(CapabilityStage { - capability: String::from("SOUND"), + capability: Path::WellKnown(WellKnownPath::Sound), outputs: vec![ty!(i16[16000])], args: map! { hz: "16000".into() }, }), @@ -1122,7 +1161,7 @@ pipeline: hz: 16000 "#; let should_be = Stage::Capability(CapabilityStage { - capability: String::from("SOUND"), + capability: Path::WellKnown(WellKnownPath::Sound), outputs: vec![Type { name: String::from("i16"), dimensions: vec![16000], diff --git a/crates/rune-cli/Cargo.toml b/crates/rune-cli/Cargo.toml index 7dff90ef5b..273fc33348 100644 --- a/crates/rune-cli/Cargo.toml +++ b/crates/rune-cli/Cargo.toml @@ -34,8 +34,10 @@ image = "0.23.14" indexmap = "1.6.2" once_cell = "1.7.0" query_based_compiler = { version = "0.11.3", path = "../compiler", package = "hotg-rune-compiler" } +queryst = "2.1.0" rand = "0.8.3" regex = "1.5.4" +reqwest = { version = "0.11.10", features = ["blocking"] } salsa = "0.16.1" serde = { version = "1.0.125", features = ["derive"] } serde_json = "1.0.64" diff --git a/crates/rune-cli/benches/rune_benchmark.rs b/crates/rune-cli/benches/rune_benchmark.rs index 61af82ad11..ee2e8df37b 100644 --- a/crates/rune-cli/benches/rune_benchmark.rs +++ b/crates/rune-cli/benches/rune_benchmark.rs @@ -25,7 +25,9 @@ pub fn project_root() -> PathBuf { .to_path_buf() } -pub fn example_dir() -> PathBuf { project_root().join("examples") } +pub fn example_dir() -> PathBuf { + project_root().join("examples") +} fn load_rune(path: PathBuf) -> Vec { std::fs::read(&path) diff --git a/crates/rune-cli/src/build/abi_v1.rs b/crates/rune-cli/src/build/abi_v1.rs index 5aa6b59e7d..f8e9439123 100644 --- a/crates/rune-cli/src/build/abi_v1.rs +++ b/crates/rune-cli/src/build/abi_v1.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, sync::Arc}; +use std::{convert::TryInto, fmt::Display, path::PathBuf, sync::Arc}; use anyhow::{Context, Error}; use query_based_compiler::{ @@ -9,6 +9,7 @@ use query_based_compiler::{ ReadError, }; use salsa::Storage; +use serde::Deserialize; use uriparse::{Scheme, URI}; use crate::{Build, Unstable}; @@ -59,11 +60,14 @@ impl salsa::Database for Database {} impl FileSystem for Database { fn read(&self, path: &URI<'_>) -> Result, ReadError> { match path.scheme() { - Scheme::FileSystem => read_file(path.path()), + Scheme::FileSystem | Scheme::File => read_file(path.path()), + Scheme::HTTP | Scheme::HTTPS => download_from_the_internet(path), Scheme::Unregistered(u) if u.as_str().is_empty() => { read_file(path.path()) }, - + Scheme::Unregistered(u) if u.as_str() == "wapm" => { + download_from_wapm(path) + }, other => Err(ReadError::UnsupportedScheme { scheme: other.as_str().into(), }), @@ -86,3 +90,58 @@ fn read_file(path: &uriparse::Path<'_>) -> Result, ReadError> { .map(Vector::from) .map_err(|e| ReadError::Other(Arc::new(e) as Arc<_>)) } + +fn download_from_wapm(uri: &URI<'_>) -> Result, ReadError> { + let (namespace, package_name) = match uri.path().segments() { + [ns, pkg] => (ns.as_str(), pkg.as_str()), + _ => { + return Err(ReadError::other(MalformedPackagePath { + path: uri.path().clone().into_owned(), + })) + }, + }; + + // https://registry-cdn.wapm.io/contents/hotg-ai/softmax/0.12.0/softmax.wasm + let version = uri + .query() + .and_then(|q| queryst::parse(q.as_str()).ok()) + .and_then(|p| QueryParams::deserialize(&p).ok()) + .and_then(|q| q.version) + .ok_or_else(|| { + ReadError::msg("Unable to determine the version number") + })?; + + let wapm_url = format!("https://registry-cdn.wapm.io/contents/{namespace}/{package_name}/{version}/{package_name}.wasm"); + let wapm_url = wapm_url.as_str().try_into().map_err(ReadError::other)?; + + download_from_the_internet(&wapm_url) +} + +#[tracing::instrument] +fn download_from_the_internet(uri: &URI<'_>) -> Result, ReadError> { + let url = uri.to_string(); + let body = reqwest::blocking::get(&url) + .and_then(|response| response.error_for_status()) + .and_then(|response| response.bytes()) + .map_err(ReadError::other)?; + + Ok(body.as_ref().into()) +} + +#[derive(Debug, serde::Deserialize)] +struct QueryParams { + version: Option, +} + +#[derive(Debug)] +struct MalformedPackagePath { + path: uriparse::Path<'static>, +} + +impl Display for MalformedPackagePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Unable to determine the package name and namespace from \"{}\". Expected something like /", self.path) + } +} + +impl std::error::Error for MalformedPackagePath {} diff --git a/examples/sine/Runefile.abi-v1.yml b/examples/sine/Runefile.abi-v1.yml new file mode 100644 index 0000000000..fc76d9a385 --- /dev/null +++ b/examples/sine/Runefile.abi-v1.yml @@ -0,0 +1,36 @@ +version: 1 +image: runicos/base +pipeline: + rand: + capability: RAW + args: + length: 4 + outputs: + - type: F32 + dimensions: + - 1 + - 1 + mod360: + proc-block: wapm:///hotg-ai/modulo?version=0.11.3 + inputs: + - rand + outputs: + - type: F32 + dimensions: + - 1 + - 1 + args: + modulus: 360.0 + sine: + model: "./sinemodel.tflite" + inputs: + - mod360 + outputs: + - type: F32 + dimensions: + - 1 + - 1 + serial: + out: serial + inputs: + - sine From fbd50bc849622f470aa72b0fa5da17634ab23229 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 19 May 2022 19:06:56 +0800 Subject: [PATCH 40/84] Patch capability paths to use local proc-block files --- crates/compiler/src/codegen/query.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index d5f083e564..d8a9127e47 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -87,7 +87,12 @@ fn patch_paths( tracing::debug!(new=%path, old=%p.proc_block, "Patching proc-block path"); p.proc_block = path; }, - Stage::Capability(_) | Stage::Out(_) => {}, + Stage::Capability(c) => { + let path = Path::FileSystem(format!("proc_blocks/{name}")); + tracing::debug!(new=%path, old=%c.capability, "Patching capability path"); + c.capability = path; + }, + Stage::Out(_) => {}, } } From d3b6449f32c883f109c6cf52dc274815abae5f77 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 19 May 2022 19:57:29 +0800 Subject: [PATCH 41/84] Forgot to add capability proc-blocks to the final ZIP file --- crates/compiler/src/parse/query.rs | 37 ++++++++++++++++++----------- crates/rune-cli/src/build/abi_v1.rs | 16 ++++++------- rust-toolchain.toml | 2 +- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index 052d21e114..e03dbee7c6 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -5,9 +5,9 @@ use uriparse::{URIBuilder, URIError, URI}; use crate::{ im::{OrdMap, Vector}, parse::{ - Document, DocumentV1, ItemType, ModelStage, NotFound, ParseFailed, - Path, ProcBlockStage, ResourceDeclaration, Stage, WellKnownPath, - WrongItemType, + CapabilityStage, Document, DocumentV1, ItemType, ModelStage, NotFound, + ParseFailed, Path, ProcBlockStage, ResourceDeclaration, Stage, + WellKnownPath, WrongItemType, }, BuildConfig, Environment, FileSystem, Text, }; @@ -157,9 +157,12 @@ fn proc_blocks( let mut proc_blocks = BTreeMap::default(); for (name, stage) in &doc.pipeline { - if let Stage::ProcBlock(_) = stage { - let binary = db.proc_block(name.into())?; - proc_blocks.insert(Text::from(name), binary); + match stage { + Stage::Capability(_) | Stage::ProcBlock(_) => { + let binary = db.proc_block(name.into())?; + proc_blocks.insert(Text::from(name), binary); + }, + _ => {}, } } @@ -182,10 +185,14 @@ fn proc_block( }) .map_err(|e| Arc::new(e) as crate::Error)?; - if let Stage::ProcBlock(ProcBlockStage { proc_block, .. }) = stage { - read(db, &proc_block) - } else { - todo!() + match stage { + Stage::Capability(CapabilityStage { + capability: path, .. + }) + | Stage::ProcBlock(ProcBlockStage { + proc_block: path, .. + }) => read(db, path), + _ => todo!(), } } @@ -287,10 +294,12 @@ fn file_uri(db: &dyn Frontend, path: &Path) -> Result, URIError> { fn wapm_uri(w: WellKnownPath) -> URI<'static> { let uri = match w { - WellKnownPath::Accel => "wapm:///hotg-ai/accelerometer-input", - WellKnownPath::Image => "wapm:///hotg-ai/image-input", - WellKnownPath::Raw => "wapm:///hotg-ai/tensor-input", - WellKnownPath::Sound => "wapm:///hotg-ai/sound-input", + WellKnownPath::Accel => { + "wapm:///hotg-ai/accelerometer_input?version=0.12.0" + }, + WellKnownPath::Image => "wapm:///hotg-ai/image_input?version=0.12.0", + WellKnownPath::Raw => "wapm:///hotg-ai/tensor_input?version=0.12.0", + WellKnownPath::Sound => "wapm:///hotg-ai/sound_input?version=0.12.0", }; uri.try_into().expect("Should never fail") diff --git a/crates/rune-cli/src/build/abi_v1.rs b/crates/rune-cli/src/build/abi_v1.rs index f8e9439123..1c24b4eef4 100644 --- a/crates/rune-cli/src/build/abi_v1.rs +++ b/crates/rune-cli/src/build/abi_v1.rs @@ -91,6 +91,7 @@ fn read_file(path: &uriparse::Path<'_>) -> Result, ReadError> { .map_err(|e| ReadError::Other(Arc::new(e) as Arc<_>)) } +#[tracing::instrument] fn download_from_wapm(uri: &URI<'_>) -> Result, ReadError> { let (namespace, package_name) = match uri.path().segments() { [ns, pkg] => (ns.as_str(), pkg.as_str()), @@ -102,14 +103,13 @@ fn download_from_wapm(uri: &URI<'_>) -> Result, ReadError> { }; // https://registry-cdn.wapm.io/contents/hotg-ai/softmax/0.12.0/softmax.wasm - let version = uri - .query() - .and_then(|q| queryst::parse(q.as_str()).ok()) - .and_then(|p| QueryParams::deserialize(&p).ok()) - .and_then(|q| q.version) - .ok_or_else(|| { - ReadError::msg("Unable to determine the version number") - })?; + let query = uri.query().map(|q| q.as_str()).unwrap_or_default(); + let query = queryst::parse(query).map_err(|e| ReadError::msg(e.message))?; + let query_params: QueryParams = + QueryParams::deserialize(&query).map_err(ReadError::other)?; + let version = query_params + .version + .ok_or_else(|| ReadError::msg("No version specified"))?; let wapm_url = format!("https://registry-cdn.wapm.io/contents/{namespace}/{package_name}/{version}/{package_name}.wasm"); let wapm_url = wapm_url.as_str().try_into().map_err(ReadError::other)?; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 588734290d..98b6ff1571 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2022-04-28" +channel = "nightly-2022-05-17" targets = ["wasm32-unknown-unknown"] components = ["rustfmt"] From 00badc90deeb50b44b2458cb7e45939e859ff334 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 19 May 2022 20:02:33 +0800 Subject: [PATCH 42/84] Updated a test --- crates/compiler/src/codegen/query.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index d8a9127e47..231b3106a3 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -264,7 +264,12 @@ mod tests { entries.sort(); assert_eq!( entries, - &["Runefile.yml", "models/sine", "proc_blocks/mod360",] + &[ + "Runefile.yml", + "models/sine", + "proc_blocks/mod360", + "proc_blocks/rand" + ] ); let expected = r#" @@ -272,7 +277,7 @@ mod tests { image: runicos/base pipeline: rand: - capability: RAW + capability: proc_blocks/rand outputs: - type: F32 dimensions: From ec5fa27f7f0ab263f23f0667f282f7ca0571cbb6 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 24 May 2022 11:30:15 +0800 Subject: [PATCH 43/84] Fixed how things are logged during integration tests --- Cargo.lock | 1 - crates/rune-cli/src/bin/rune.rs | 2 ++ integration-tests/Cargo.toml | 1 - integration-tests/src/lib.rs | 23 ++++++++++++++++++----- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ad883655d..4a6fe636ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1443,7 +1443,6 @@ dependencies = [ "env_logger", "log", "once_cell", - "rayon", "regex 1.5.5", "structopt", "walkdir", diff --git a/crates/rune-cli/src/bin/rune.rs b/crates/rune-cli/src/bin/rune.rs index a2e26bb5c4..3f00ea32ae 100644 --- a/crates/rune-cli/src/bin/rune.rs +++ b/crates/rune-cli/src/bin/rune.rs @@ -21,6 +21,7 @@ fn main() -> Result<(), Error> { "cranelift_codegen=warn", "regalloc=warn", "salsa=warn", + "wasmer_compiler_cranelift=warn", ] .join(","), ); @@ -35,6 +36,7 @@ fn main() -> Result<(), Error> { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) + .with_writer(std::io::stderr) .init(); match cmd { diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 90a9f1a3d0..915b78c849 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -15,7 +15,6 @@ anyhow = "1.0.52" env_logger = "0.9.0" log = "0.4.14" once_cell = "1.9.0" -rayon = "1.5.2" regex = "1.5.4" structopt = "0.3.25" walkdir = "2.3.2" diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs index 5e1bfca45f..d9e5fcc8d0 100644 --- a/integration-tests/src/lib.rs +++ b/integration-tests/src/lib.rs @@ -10,7 +10,6 @@ use std::{ }; use anyhow::{Context, Error}; -use rayon::prelude::*; pub use crate::loader::{Category, ExitCondition, FullName, Test}; @@ -30,7 +29,7 @@ pub struct TestSuite { impl TestSuite { pub fn run(&self, ctx: &TestContext, cb: &dyn Callbacks) { - self.tests.par_iter().for_each(|test| { + self.tests.iter().for_each(|test| { let name = &test.name; if !cb.should_run(name) { @@ -56,7 +55,9 @@ pub trait Callbacks: Sync { fn on_bug(&self, name: &FullName, error: Error); fn on_fail(&self, name: &FullName, errors: Vec, output: Output); /// Should this test be executed? - fn should_run(&self, _name: &FullName) -> bool { true } + fn should_run(&self, _name: &FullName) -> bool { + true + } } #[derive(Debug)] @@ -137,7 +138,17 @@ impl TestContext { .arg("--rune-repo-dir") .arg(&self.rune_project_dir); - cmd.env("RUST_LOG", "debug,wasmer_compiler_cranelift=warn"); + // Note: We only want to see debug log messages from our own crates. + // This also gives us a consistent RUST_LOG to work with. + const WHITELIST: &[&str] = + &["hotg_rune_cli", "hotg_rune_compiler", "hotg_rune_runtime"]; + let rust_log = WHITELIST + .iter() + .map(|pkg| format!("{pkg}=debug")) + .collect::>() + .join(","); + + cmd.env("RUST_LOG", format!("warn,{rust_log}")); cmd } @@ -156,7 +167,9 @@ impl TestContext { struct CommandOutput(Output); impl CommandOutput { - fn new(output: Output) -> Self { CommandOutput(output) } + fn new(output: Output) -> Self { + CommandOutput(output) + } } impl Display for CommandOutput { From cd9b2aebb02f03306666dd92b62f6123fa8d923b Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 15:15:20 +0530 Subject: [PATCH 44/84] In Progress: Implementing Zune Engine as a webassembly engine --- .gitmodules | 3 + Cargo.lock | 438 ++++++++++++++++++------------ crates/compiler/src/parse/mod.rs | 2 +- crates/rune-cli/Cargo.toml | 2 +- crates/rune-cli/src/run.rs | 7 + crates/runtime/Cargo.toml | 92 +++++-- crates/runtime/src/engine/mod.rs | 5 +- crates/runtime/src/engine/zune.rs | 418 ++++++++++++++++++++++++++++ crates/runtime/src/runtime.rs | 5 + wit-files | 1 + 10 files changed, 764 insertions(+), 209 deletions(-) create mode 100644 .gitmodules create mode 100644 crates/runtime/src/engine/zune.rs create mode 160000 wit-files diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..63b2e8eb16 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "wit-files"] + path = wit-files + url = git@github.com:hotg-ai/wit-files.git diff --git a/Cargo.lock b/Cargo.lock index 4a6fe636ad..91b5b7a137 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,7 +127,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide 0.5.1", + "miniz_oxide 0.5.3", "object", "rustc-demangle", ] @@ -170,7 +170,7 @@ dependencies = [ "peeking_take_while", "proc-macro2", "quote", - "regex 1.5.5", + "regex 1.5.6", "rustc-hash", "shlex 1.1.0", "which", @@ -259,7 +259,7 @@ checksum = "c1a56f0765f909b7b4d7d5c7833e6c66bc1f63dab8046adc8effd568d676cd04" dependencies = [ "chrono", "derive_more", - "semver 1.0.7", + "semver 1.0.9", "serde", ] @@ -294,9 +294,9 @@ checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" [[package]] name = "bytecheck" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "314889ea31cda264cb7c3d6e6e5c9415a987ecb0e72c17c00d36fbb881d34abe" +checksum = "3a31f923c2db9513e4298b72df143e6e655a759b3d6a0966df18f81223fff54f" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a2b3b92c135dae665a6f760205b89187638e83bed17ef3e44e83c712cf30600" +checksum = "edb17c862a905d912174daa27ae002326fff56dc8b8ada50a0a5f0976cb174f0" dependencies = [ "proc-macro2", "quote", @@ -360,9 +360,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3132262930b0522068049f5870a856ab8affc80c70d08b6ecb785771a6fc23" +checksum = "869119e97797867fd90f5e22af7d0bd274bd4635ebb9eb68c04f3f513ae6c412" dependencies = [ "serde", ] @@ -478,9 +478,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" +checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b" dependencies = [ "glob", "libc", @@ -674,7 +674,7 @@ dependencies = [ "oorandom", "plotters", "rayon", - "regex 1.5.5", + "regex 1.5.6", "serde", "serde_cbor", "serde_derive", @@ -996,7 +996,7 @@ dependencies = [ "atty", "humantime", "log", - "regex 1.5.5", + "regex 1.5.6", "termcolor", ] @@ -1022,7 +1022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d95b4efe5be9104a4a18a9916e86654319895138be727b229820c39257c30dda" dependencies = [ "bit-set", - "regex 1.5.5", + "regex 1.5.6", ] [[package]] @@ -1036,14 +1036,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" dependencies = [ - "cfg-if", "crc32fast", - "libc", - "miniz_oxide 0.5.1", + "miniz_oxide 0.5.3", ] [[package]] @@ -1242,7 +1240,7 @@ dependencies = [ "bstr", "fnv", "log", - "regex 1.5.5", + "regex 1.5.6", ] [[package]] @@ -1281,9 +1279,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" dependencies = [ "ahash", ] @@ -1348,7 +1346,7 @@ dependencies = [ "predicates", "queryst", "rand 0.8.5", - "regex 1.5.5", + "regex 1.5.6", "reqwest", "salsa", "serde", @@ -1373,7 +1371,7 @@ dependencies = [ "jsonschema", "once_cell", "pretty_assertions 1.2.1", - "regex 1.5.5", + "regex 1.5.6", "reqwest", "salsa", "schemars", @@ -1408,7 +1406,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "regex 1.5.5", + "regex 1.5.6", "schemars", "serde", "serde_json", @@ -1443,7 +1441,7 @@ dependencies = [ "env_logger", "log", "once_cell", - "regex 1.5.5", + "regex 1.5.6", "structopt", "walkdir", ] @@ -1489,10 +1487,12 @@ version = "0.11.3" dependencies = [ "anyhow", "csv", + "hotg-rune-compiler 0.11.3", "hotg-rune-core 0.11.3", "hotg-runecoral", "hound", "image", + "indexmap", "log", "rand 0.8.5", "serde", @@ -1502,13 +1502,15 @@ dependencies = [ "wasm3", "wasmer", "wasmparser 0.83.0", + "wit-bindgen-wasmer", + "zip 0.6.2", ] [[package]] name = "hotg-runecoral" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0005a9686a51972724e64da4ace8e9599a5bae4c4e3a4852b5bcf4d2112c4464" +checksum = "b316b99f82928c84fba9a8c4b7e4fd999cfd6b8519b9a18ed78ed4999102cb60" dependencies = [ "bindgen", "bitflags", @@ -1542,14 +1544,14 @@ checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", - "itoa 1.0.1", + "itoa 1.0.2", ] [[package]] name = "http-body" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -1591,9 +1593,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.18" +version = "0.14.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" dependencies = [ "bytes", "futures-channel", @@ -1604,7 +1606,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.1", + "itoa 1.0.2", "pin-project-lite", "socket2", "tokio", @@ -1626,6 +1628,12 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + [[package]] name = "ident_case" version = "1.0.1" @@ -1675,9 +1683,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" dependencies = [ "autocfg", "hashbrown 0.11.2", @@ -1692,9 +1700,9 @@ checksum = "05a0bd019339e5d968b37855180087b7b9d512c5046fbd244cf8c95687927d6e" [[package]] name = "insta" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "689960f187c43c01650c805fb6bc6f55ab944499d86d4ffe9474ad78991d8e94" +checksum = "bcc3e639bcba360d9237acabd22014c16f3df772db463b7446cd81b070714767" dependencies = [ "console", "once_cell", @@ -1745,9 +1753,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" [[package]] name = "jobserver" @@ -1789,13 +1797,13 @@ dependencies = [ "fancy-regex", "fraction", "iso8601", - "itoa 1.0.1", + "itoa 1.0.2", "lazy_static", "memchr", "num-cmp", - "parking_lot 0.12.0", + "parking_lot 0.12.1", "percent-encoding 2.1.0", - "regex 1.5.5", + "regex 1.5.6", "reqwest", "serde", "serde_json", @@ -1858,9 +1866,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.124" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libgit2-sys" @@ -1886,9 +1894,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.6" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", "libc", @@ -1914,9 +1922,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", "serde", @@ -1980,9 +1988,9 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" @@ -2035,34 +2043,23 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys", ] [[package]] @@ -2105,15 +2102,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - [[package]] name = "num" version = "0.2.1" @@ -2168,9 +2156,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -2212,9 +2200,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -2231,18 +2219,18 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.28.3" +version = "0.28.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" dependencies = [ "crc32fast", "hashbrown 0.11.2", @@ -2252,9 +2240,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "oorandom" @@ -2270,18 +2258,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -2290,9 +2290,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -2307,7 +2307,7 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3df761f6470298359f84fcfb60d86db02acc22c251c37265c07a3d1057d2389" dependencies = [ - "regex 1.5.5", + "regex 1.5.6", ] [[package]] @@ -2332,12 +2332,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.2", + "parking_lot_core 0.9.3", ] [[package]] @@ -2356,9 +2356,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -2498,7 +2498,7 @@ dependencies = [ "itertools", "normalize-line-endings", "predicates-core", - "regex 1.5.5", + "regex 1.5.6", ] [[package]] @@ -2573,11 +2573,11 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -2600,6 +2600,17 @@ dependencies = [ "syn", ] +[[package]] +name = "pulldown-cmark" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + [[package]] name = "pulldown-cmark" version = "0.9.1" @@ -2694,9 +2705,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg", "crossbeam-deque", @@ -2706,9 +2717,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -2771,13 +2782,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick 0.7.18", "memchr", - "regex-syntax 0.6.25", + "regex-syntax 0.6.26", ] [[package]] @@ -2786,7 +2797,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax 0.6.25", + "regex-syntax 0.6.26", ] [[package]] @@ -2800,9 +2811,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "region" @@ -2872,12 +2883,12 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.37" +version = "0.7.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f08c8062c1fe1253064043b8fc07bfea1b9702b71b4a86c11ea3588183b12e1" +checksum = "517a3034eb2b1499714e9d1e49b2367ad567e07639b69776d35e259d9c27cca6" dependencies = [ "bytecheck", - "hashbrown 0.12.0", + "hashbrown 0.12.1", "ptr_meta", "rend", "rkyv_derive", @@ -2886,9 +2897,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.37" +version = "0.7.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e289706df51226e84814bf6ba1a9e1013112ae29bc7a9878f73fce360520c403" +checksum = "505c209ee04111a006431abf39696e640838364d67a107c559ababaf6fd8c9dd" dependencies = [ "proc-macro2", "quote", @@ -2908,7 +2919,7 @@ dependencies = [ "libc", "log", "once_cell", - "pulldown-cmark", + "pulldown-cmark 0.9.1", "serde_json", "tracing", "tracing-subscriber", @@ -2932,7 +2943,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.7", + "semver 1.0.9", ] [[package]] @@ -2943,9 +2954,9 @@ checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" [[package]] name = "salsa" @@ -2987,19 +2998,19 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "winapi", + "windows-sys", ] [[package]] name = "schemars" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6b5a3c80cea1ab61f4260238409510e814e38b4b563c06044edf91e7dc070e3" +checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed" dependencies = [ "dyn-clone", "indexmap", @@ -3010,9 +3021,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ae4dce13e8614c46ac3c38ef1c0d668b101df6ac39817aebdaa26642ddae9b" +checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743" dependencies = [ "proc-macro2", "quote", @@ -3079,9 +3090,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" dependencies = [ "serde", ] @@ -3097,9 +3108,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] @@ -3116,9 +3127,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" +checksum = "212e73464ebcde48d723aa02eb270ba62eff38a9b732df31f33f1b4e145f3a54" dependencies = [ "serde", ] @@ -3135,9 +3146,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -3146,9 +3157,9 @@ dependencies = [ [[package]] name = "serde_derive_internals" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dbab34ca63057a1f15280bdf3c39f2b1eb1b54c17e98360e511637aef7418c6" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ "proc-macro2", "quote", @@ -3157,11 +3168,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] @@ -3173,16 +3184,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.1", + "itoa 1.0.2", "ryu", "serde", ] [[package]] name = "serde_yaml" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" +checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" dependencies = [ "indexmap", "ryu", @@ -3326,20 +3337,20 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.91" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] name = "target-lexicon" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" +checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" [[package]] name = "tempdir" @@ -3401,18 +3412,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -3465,7 +3476,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.2", "libc", "num_threads", "time-macros", @@ -3504,9 +3515,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.18.0" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ "bytes", "libc", @@ -3531,9 +3542,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" dependencies = [ "bytes", "futures-core", @@ -3612,7 +3623,7 @@ dependencies = [ "ansi_term", "lazy_static", "matchers", - "regex 1.5.5", + "regex 1.5.6", "sharded-slab", "smallvec", "thread_local 1.1.4", @@ -3683,6 +3694,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + [[package]] name = "unicode-normalization" version = "0.1.19" @@ -3706,9 +3723,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "uriparse" @@ -4123,9 +4140,9 @@ checksum = "718ed7c55c2add6548cca3ddd6383d738cd73b892df400e96b9aa876f0141d7a" [[package]] name = "wast" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb4f48a8b083dbc50e291e430afb8f524092bb00428957bcc63f49f856c64ac" +checksum = "f882898b8b817cc4edc16aa3692fdc087b356edc8cc0c2164f5b5181e31c3870" dependencies = [ "leb128", "memchr", @@ -4134,9 +4151,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.42" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0401b6395ce0db91629a75b29597ccb66ea29950af9fc859f1bb3a736609c76e" +checksum = "48b3b9b3e39e66c7fd3f8be785e74444d216260f491e93369e317ed6482ff80f" dependencies = [ "wast", ] @@ -4201,9 +4218,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -4214,33 +4231,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "winreg" @@ -4251,6 +4268,69 @@ dependencies = [ "winapi", ] +[[package]] +name = "wit-bindgen-gen-core" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wit-bindgen?branch=wasmer#8772835affd9aa1cff1831e2a5f822d832006001" +dependencies = [ + "anyhow", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-gen-rust" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wit-bindgen?branch=wasmer#8772835affd9aa1cff1831e2a5f822d832006001" +dependencies = [ + "heck 0.3.3", + "wit-bindgen-gen-core", +] + +[[package]] +name = "wit-bindgen-gen-wasmer" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wit-bindgen?branch=wasmer#8772835affd9aa1cff1831e2a5f822d832006001" +dependencies = [ + "heck 0.3.3", + "wit-bindgen-gen-core", + "wit-bindgen-gen-rust", +] + +[[package]] +name = "wit-bindgen-wasmer" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wit-bindgen?branch=wasmer#8772835affd9aa1cff1831e2a5f822d832006001" +dependencies = [ + "anyhow", + "bitflags", + "thiserror", + "wasmer", + "wit-bindgen-wasmer-impl", +] + +[[package]] +name = "wit-bindgen-wasmer-impl" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wit-bindgen?branch=wasmer#8772835affd9aa1cff1831e2a5f822d832006001" +dependencies = [ + "proc-macro2", + "syn", + "wit-bindgen-gen-core", + "wit-bindgen-gen-wasmer", +] + +[[package]] +name = "wit-parser" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wit-bindgen?branch=wasmer#8772835affd9aa1cff1831e2a5f822d832006001" +dependencies = [ + "anyhow", + "id-arena", + "pulldown-cmark 0.8.0", + "unicode-normalization", + "unicode-xid", +] + [[package]] name = "xtask" version = "0.0.0" @@ -4321,18 +4401,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.10.0+zstd.1.5.2" +version = "0.10.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b1365becbe415f3f0fcd024e2f7b45bacfb5bdd055f0dc113571394114e7bdd" +checksum = "5f4a6bd64f22b5e3e94b4e238669ff9f10815c27a5180108b849d24174a83847" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.4+zstd.1.5.2" +version = "4.1.6+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7cd17c9af1a4d6c24beb1cc54b17e2ef7b593dc92f19e9d9acad8b182bbaee" +checksum = "94b61c51bb270702d6167b8ce67340d2754b088d0c091b06e593aa772c3ee9bb" dependencies = [ "libc", "zstd-sys", diff --git a/crates/compiler/src/parse/mod.rs b/crates/compiler/src/parse/mod.rs index 24191fc66e..d94e3777ec 100644 --- a/crates/compiler/src/parse/mod.rs +++ b/crates/compiler/src/parse/mod.rs @@ -4,7 +4,7 @@ //! type. mod query; -mod yaml; +pub mod yaml; use std::sync::Arc; diff --git a/crates/rune-cli/Cargo.toml b/crates/rune-cli/Cargo.toml index 273fc33348..0ca73c24a4 100644 --- a/crates/rune-cli/Cargo.toml +++ b/crates/rune-cli/Cargo.toml @@ -26,7 +26,7 @@ dotenv = "0.15.0" hotg-rune-compiler = "^0.11.0" hotg-rune-core = "^0.11.0" hotg-rune-proc-blocks = { version = "0.11.3", path = "../proc-blocks" } -hotg-rune-runtime = { path = "../runtime", version = "^0.11.0", features = ["builtins", "wasm3", "wasmer"] } +hotg-rune-runtime = { path = "../runtime", version = "^0.11.0", features = ["builtins", "wasm3", "wasmer", "zune"] } hotg-runecoral = "0.3.11" hound = "3.4.0" human-panic = "1.0.3" diff --git a/crates/rune-cli/src/run.rs b/crates/rune-cli/src/run.rs index be547ee18e..23864c5d1e 100644 --- a/crates/rune-cli/src/run.rs +++ b/crates/rune-cli/src/run.rs @@ -9,6 +9,7 @@ use once_cell::sync::Lazy; use regex::Regex; use structopt::StructOpt; use strum::VariantNames; +use std::ffi::OsStr; #[derive(Debug, Clone, PartialEq, StructOpt)] pub struct Run { @@ -76,6 +77,10 @@ impl Run { format!("Unable to read \"{}\"", self.rune.display()) })?; + if self.rune.extension().unwrap_or(OsStr::new("")).to_ascii_lowercase() == "zune" { + self.engine = Engine::Zune; + } + let mut runtime: Runtime = self .load_runtime(&rune) .context("Unable to load the Runtime")?; @@ -172,6 +177,7 @@ impl Run { match self.engine { Engine::Wasm3 => Runtime::wasm3(rune), Engine::Wasmer => Runtime::wasmer(rune), + Engine::Zune => Runtime::zune(rune), } } @@ -252,4 +258,5 @@ fn parse_key_value_pair(s: &str) -> Result<(&str, &str), Error> { enum Engine { Wasm3, Wasmer, + Zune } diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 522cb410f8..f12963ef38 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -1,44 +1,82 @@ [package] -name = "hotg-rune-runtime" -version = "0.11.3" -edition = "2018" authors = ["The Rune Developers "] -license = "MIT OR Apache-2.0" -homepage = "https://hotg.dev/" -repository = "https://github.com/hotg-ai/rune" categories = ["science", "wasm"] -keywords = ["rune", "runtime", "tinyml", "machine", "learning"] description = "Common abstractions and utilities used by Rune runtimes." +edition = "2018" +homepage = "https://hotg.dev/" +keywords = ["rune", "runtime", "tinyml", "machine", "learning"] +license = "MIT OR Apache-2.0" +name = "hotg-rune-runtime" readme = "README.md" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - +repository = "https://github.com/hotg-ai/rune" +version = "0.11.3" [dependencies] anyhow = "1.0.40" -csv = { version = "1.1.6", optional = true } -hotg-rune-core = { path = "../rune-core", version = "^0.11.0", features = ["std"] } -hotg-runecoral = { version = "0.3.11", optional = true } -hound = { version = "3.4.0", optional = true } -image = { version = "0.23.14", optional = true } log = "0.4.14" -rand = { version = "0.8.3", optional = true } -serde = { version = "1.0.136", features = ["derive"] } -serde_json = { version = "1.0.79" } thiserror = "1.0.30" -wasm3 = { git = "https://github.com/wasm3/wasm3-rs", optional = true } -wasmer = { version = "2.2.0-rc2", optional = true } wasmparser = "0.83.0" +wit-bindgen-wasmer = { git = "https://github.com/wasmerio/wit-bindgen", branch = "wasmer" } +indexmap = { version = "1.8.0", features = ["serde-1"] } + +[dependencies.csv] +optional = true +version = "1.1.6" + +[dependencies.hotg-rune-compiler] +path = "../compiler" +version = "^0.11.0" + +[dependencies.hotg-rune-core] +features = ["std"] +path = "../rune-core" +version = "^0.11.0" + +[dependencies.hotg-runecoral] +optional = true +version = "0.3.11" + +[dependencies.hound] +optional = true +version = "3.4.0" + +[dependencies.image] +optional = true +version = "0.23.14" + +[dependencies.rand] +optional = true +version = "0.8.3" + +[dependencies.serde] +features = ["derive"] +version = "1.0.136" + +[dependencies.serde_json] +version = "1.0.79" + +[dependencies.wasm3] +git = "https://github.com/wasm3/wasm3-rs" +optional = true + +[dependencies.wasmer] +optional = true +version = "2.2.0-rc2" + +[dependencies.zip] +optional = true +version = "0.6.2" + +[dev-dependencies] +tempfile = "3.2.0" [features] -default = ["builtins", "tflite"] builtins = ["hound", "image", "rand", "rand/small_rng", "csv"] +default = ["builtins", "tflite"] tflite = ["hotg-runecoral"] -# Enable rustdoc's "This is supported on crate feature XXX only" annotations -# (requires nightly) unstable_doc_cfg = [] +zune = ["wasmer", "zip"] -[dev-dependencies] -tempfile = "3.2.0" - -[package.metadata.docs.rs] +[metadata] +[metadata.docs] +[metadata.docs.rs] all-features = true diff --git a/crates/runtime/src/engine/mod.rs b/crates/runtime/src/engine/mod.rs index 798f0a9465..8ca19d82d9 100644 --- a/crates/runtime/src/engine/mod.rs +++ b/crates/runtime/src/engine/mod.rs @@ -3,6 +3,8 @@ mod host_functions; mod wasm3; #[cfg(feature = "wasmer")] mod wasmer; +#[cfg(feature = "zune")] +mod zune; use std::sync::Arc; @@ -12,7 +14,8 @@ use anyhow::Error; pub(crate) use self::wasm3::Wasm3Engine; #[cfg(feature = "wasmer")] pub(crate) use self::wasmer::WasmerEngine; - +#[cfg(feature = "zune")] +pub(crate) use self::zune::ZuneEngine; /// A WebAssembly virtual machine that links Rune with pub(crate) trait WebAssemblyEngine { fn load( diff --git a/crates/runtime/src/engine/zune.rs b/crates/runtime/src/engine/zune.rs new file mode 100644 index 0000000000..c275783c9d --- /dev/null +++ b/crates/runtime/src/engine/zune.rs @@ -0,0 +1,418 @@ +use std::{ + convert::TryInto, + fmt::{self, Display, Formatter}, + sync::{Arc, Mutex}, + io::{Cursor, Read}, + collections::HashMap, +}; + +use zip; +use anyhow::{Context, Error, anyhow}; +use hotg_rune_core::{ElementType as RuneElementType, Shape, TFLITE_MIMETYPE}; +use indexmap::IndexMap; +use wasmer::{ + Array, Function, Instance, LazyInit, Memory, Module, NativeFunc, + RuntimeError, Store, ValueType, WasmPtr, WasmerEnv, +}; + +use hotg_runecoral::{ + AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, Tensor, + TensorDescriptor, TensorMut, +}; + +use hotg_rune_compiler::{ + parse::yaml::*, + diagnostics::Diagnostics +}; + +use crate::{ + callbacks::Callbacks, + engine::{host_functions::HostFunctions, LoadError, WebAssemblyEngine}, +}; + +use self::{proc_block_v1::ProcBlockV1, runtime_v1::*}; + +wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); +wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); + +#[derive(Debug, Default, Clone, wasmer::WasmerEnv)] +struct Runtime(Arc>); + +#[derive(Debug, Default)] +pub struct State { + pub arguments: HashMap, + pub inputs: HashMap, + pub outputs: HashMap, +} + +pub struct ModelContext { + inference_context: InferenceContext +} + +pub struct ProcBlockContext { + instance: Instance +} + +pub struct ZuneEngine { + tensors: Vec>, + inputs: Vec, + input_tensors: HashMap, + output_tensors: HashMap, + outputs: Vec, + model_contexts: HashMap, + procblock_contexts: HashMap, + pipeline: IndexMap, + processing_order: Vec + // resources +} + +impl WebAssemblyEngine for ZuneEngine { + fn load( + binary: &[u8], + _: Arc, + ) -> Result + where + Self: Sized, + { + let mut archive = zip::ZipArchive::new(Cursor::new(binary)).context("Unable to load Zune")?; + + let mut read_zip_resource_by_path = |path: &str| -> Result, LoadError> { + let mut requested_file = archive.by_name(path).context(anyhow!("Unable to find {} in zune", path))?; + let mut buffer = Vec::new(); + requested_file.read_to_end(&mut buffer).context(anyhow!("Unable to read {} from zune", path))?; + Ok(buffer) + }; + + let runefile = String::from_utf8(read_zip_resource_by_path("Runefile.yml")?).context("Unable to read Runefile")?; + let parsed_runefile = Document::parse(&runefile).context("Unable to parse Runefile")?; + let pipeline = &parsed_runefile.to_v1().pipeline; + + let mut model_contexts: HashMap = HashMap::new(); + let mut procblock_contexts: HashMap = HashMap::new(); + let mut inputs = Vec::new(); + let mut outputs = Vec::new(); + + for item in pipeline { + // Collect each output tensor into tensors + let stage_name = item.0; + match item.1 { + Stage::Capability(_) => { + inputs.push(stage_name.to_string()); + }, + Stage::Model(stage) => { + // Instantiating the model's inference context here because that way model_data gets deallocated once we are done with it + // This way memory usage is under control + let model_data = read_zip_resource_by_path(&stage.model.to_string()) + .context(format!("Unable to read model from zune {}", stage.model))?; + let inference_context = + InferenceContext::create_context(TFLITE_MIMETYPE, &model_data, AccelerationBackend::NONE) + .context(format!("Error Instantiating model from zune {}", stage.model))?; + + model_contexts.insert(stage_name.to_string(), ModelContext { inference_context }); + }, + Stage::ProcBlock(stage) => { + println!("Pipeline stage: {} proc block {:?} {}", stage_name, stage.args, stage.proc_block.base ); + let wasm = read_zip_resource_by_path(&stage.proc_block.base).context("Unable to load the proc_block")?; + }, + Stage::Out(_) => { + outputs.push(stage_name.to_string()); + } + } + } + + let (tensors, input_tensors, output_tensors, processing_order) + = get_tensors(&inputs, &outputs, &pipeline).context(anyhow!("Unable to map out input/output tensors"))?; + + /* + println!("input_tensors: {:?}", &input_tensors); + println!("output_tensors: {:?}", &output_tensors); + println!("processing_order: {:?}", &processing_order); + */ + + Ok(ZuneEngine { + tensors, + inputs, + input_tensors, + outputs, + output_tensors, + model_contexts, + procblock_contexts, + pipeline: pipeline.to_owned(), + processing_order + }) + } + + fn init(&mut self) -> Result<(), Error> { + //TODO: Call each proc block's graph() and each model's inputs/outputs to instantiate the tensors + + Ok(()) + } + + fn predict(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +fn get_tensors(inputs: &Vec, outputs: &Vec, pipeline: &IndexMap) + -> Result<(Vec>, HashMap, HashMap, Vec), Error> { + let mut nodes_to_visit = outputs.clone(); + let mut nodes_visited = Vec::new(); + let mut tensors: Vec> = Vec::new(); + let mut output_tensors: HashMap = HashMap::new(); + let mut input_tensors: HashMap = HashMap::new(); + + let key = |node_name: &str, tensor_index: Option| format!("{}.{}", node_name, tensor_index.or(Some(0)).unwrap()); + + // For Inputs/Capabilities - input tensors and output tensors are the same? + for item in inputs { + tensors.push(None); + input_tensors.insert(key(item, Some(0)), tensors.len() - 1); + output_tensors.insert(key(item, Some(0)), tensors.len() - 1); + } + + // Do a depth first traversal of the tree structure to determine the order of processing/calling predict() + // Also allocate the output tensors of each node along the way + while !nodes_to_visit.is_empty() { + let node = nodes_to_visit.pop().unwrap(); + nodes_visited.push(node.clone()); + + let stage = pipeline.get(&node).unwrap(); + for output_index in 0..stage.output_types().len() { + tensors.push(None); + output_tensors.insert(key(&node, Some(output_index)), tensors.len() - 1); + } + + for input in stage.inputs() { + if !nodes_visited.contains(&input.name) { + nodes_to_visit.push(input.name.clone()); + } + } + } + + // For each stage in the pipeline, since the inputs have to come from the outputs of other stages, simply map to the same tensor + for item in pipeline { + // Collect each output tensor into tensors + let stage_name = item.0; + for i in 0..item.1.inputs().len() { + let input = &item.1.inputs()[i]; + let input_key = key(&input.name, input.index); + let &input_tensor_index = output_tensors.get(&input_key).context(anyhow!("Invalid input key specified: {}", &input_key))?; + input_tensors.insert(key(stage_name, Some(i)), input_tensor_index); + } + } + + nodes_visited.reverse(); + + Ok((tensors, input_tensors, output_tensors, nodes_visited)) +} + +/* +#[derive(Debug, Clone)] +pub enum Never {} + +impl runtime_v1::RuntimeV1 for ZuneEngine { + type ArgumentHint = Never; + type ArgumentMetadata = Never; + type GraphContext = Never; + type KernelContext = Arc>; + type Metadata = Never; + type TensorHint = Never; + type TensorMetadata = Never; + type LogMetadata = Never; + + fn metadata_new(&mut self, _name: &str, _version: &str) -> Self::Metadata { + todo!() + } + fn metadata_set_description(&mut self, _self_: &Self::Metadata, _description: &str) { + todo!() + } + fn metadata_set_repository(&mut self, _self_: &Self::Metadata, _url: &str) { + todo!() + } + fn metadata_set_homepage(&mut self, _self_: &Self::Metadata, _url: &str) { + todo!() + } + fn metadata_add_tag(&mut self, _self_: &Self::Metadata, _tag: &str) { + todo!() + } + fn metadata_add_argument(&mut self, _self_: &Self::Metadata, _arg: &Self::ArgumentMetadata) { + todo!() + } + fn metadata_add_input(&mut self, _self_: &Self::Metadata, _metadata: &Self::TensorMetadata) { + todo!() + } + fn metadata_add_output(&mut self, _self_: &Self::Metadata, _metadata: &Self::TensorMetadata) { + todo!() + } + fn argument_metadata_new(&mut self, _name: &str) -> Self::ArgumentMetadata { + todo!() + } + fn argument_metadata_set_description( + &mut self, + _self_: &Self::ArgumentMetadata, + _description: &str, + ) { + todo!() + } + fn argument_metadata_set_default_value( + &mut self, + _self_: &Self::ArgumentMetadata, + _default_value: &str, + ) { + todo!() + } + fn argument_metadata_add_hint( + &mut self, + _self_: &Self::ArgumentMetadata, + _hint: &Self::ArgumentHint, + ) { + todo!() + } + fn tensor_metadata_new(&mut self, _name: &str) -> Self::TensorMetadata { + todo!() + } + fn tensor_metadata_set_description( + &mut self, + _self_: &Self::TensorMetadata, + _description: &str, + ) { + todo!() + } + fn tensor_metadata_add_hint( + &mut self, + _self_: &Self::TensorMetadata, + _hint: &Self::TensorHint, + ) { + todo!() + } + fn interpret_as_image(&mut self) -> Self::TensorHint { + todo!() + } + fn interpret_as_audio(&mut self) -> Self::TensorHint { + todo!() + } + fn supported_shapes( + &mut self, + _supported_element_types: Vec, + _dimensions: Dimensions<'_>, + ) -> Self::TensorHint { + todo!() + } + fn interpret_as_number_in_range(&mut self, _min: &str, _max: &str) -> Self::ArgumentHint { + todo!() + } + fn interpret_as_string_in_enum(&mut self, _string_enum: Vec<&str>) -> Self::ArgumentHint { + todo!() + } + fn non_negative_number(&mut self) -> Self::ArgumentHint { + todo!() + } + fn supported_argument_type(&mut self, _hint: ArgumentType) -> Self::ArgumentHint { + todo!() + } + fn register_node(&mut self, _metadata: &Self::Metadata) { + todo!() + } + fn graph_context_for_node(&mut self, _node_id: &str) -> Option { + todo!() + } + fn graph_context_get_argument( + &mut self, + _self_: &Self::GraphContext, + _name: &str, + ) -> Option { + todo!() + } + fn graph_context_add_input_tensor( + &mut self, + _self_: &Self::GraphContext, + _name: &str, + _element_type: ElementType, + _dimensions: Dimensions<'_>, + ) { + todo!() + } + fn graph_context_add_output_tensor( + &mut self, + _self_: &Self::GraphContext, + _name: &str, + _element_type: ElementType, + _dimensions: Dimensions<'_>, + ) { + todo!() + } + fn kernel_context_for_node(&mut self, _node_id: &str) -> Option { + Some(self.0.clone()) + } + fn kernel_context_get_argument( + &mut self, + state: &Arc>, + name: &str, + ) -> Option { + state.lock().unwrap().arguments.get(name).cloned() + } + fn kernel_context_get_input_tensor( + &mut self, + state: &Arc>, + name: &str, + ) -> Option { + state.lock().unwrap().inputs.get(name).cloned() + } + fn kernel_context_set_output_tensor( + &mut self, + state: &Arc>, + name: &str, + TensorParam { + element_type, + buffer, + dimensions, + }: TensorParam<'_>, + ) { + let tensor = TensorResult { + element_type, + dimensions: dimensions.iter().map(|d| d.get()).collect(), + buffer: buffer.to_vec(), + }; + state + .lock() + .unwrap() + .outputs + .insert(name.to_string(), tensor); + } + + fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { + true + } + + fn log(&mut self, metadata: LogMetadata, message: &str, data: Vec<(&'_ str, LogValue<'_>)>) { + let level = match metadata.level { + LogLevel::Trace => tracing::Level::TRACE, + LogLevel::Debug => tracing::Level::DEBUG, + LogLevel::Info => tracing::Level::INFO, + LogLevel::Warn => tracing::Level::WARN, + LogLevel::Error | LogLevel::Fatal => tracing::Level::ERROR, + }; + + let LogMetadata { + name, + target, + level: _, + file, + line, + module, + } = metadata; + + tracing::event!( + tracing::Level::INFO, + meta.level = %level, + meta.name = %name, + meta.target = target, + meta.file = file, + meta.line = line, + meta.module = module, + ?data, + message, + ); + } +} +*/ \ No newline at end of file diff --git a/crates/runtime/src/runtime.rs b/crates/runtime/src/runtime.rs index 923ceb8478..b565649b61 100644 --- a/crates/runtime/src/runtime.rs +++ b/crates/runtime/src/runtime.rs @@ -70,6 +70,11 @@ impl Runtime { Runtime::load::(rune) } + #[cfg(feature = "zune")] + pub fn zune(rune: &[u8]) -> Result { + Runtime::load::(rune) + } + fn load(rune: &[u8]) -> Result where E: WebAssemblyEngine + 'static, diff --git a/wit-files b/wit-files new file mode 160000 index 0000000000..aa6b7cf2b4 --- /dev/null +++ b/wit-files @@ -0,0 +1 @@ +Subproject commit aa6b7cf2b4d91524c630d4e32592123c3b3c93ea From 127202caf7c9db939fa75fc2116b16fffec2342a Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 15:16:11 +0530 Subject: [PATCH 45/84] Zune: Implement load() and predict() for ModelNode --- Cargo.lock | 2 + crates/runtime/Cargo.toml | 2 + crates/runtime/src/engine/zune.rs | 729 ++++++++++++++++++++++-------- 3 files changed, 555 insertions(+), 178 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91b5b7a137..3b8d13acac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1499,6 +1499,8 @@ dependencies = [ "serde_json", "tempfile", "thiserror", + "tracing", + "tracing-subscriber", "wasm3", "wasmer", "wasmparser 0.83.0", diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index f12963ef38..5e17fd430a 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -16,6 +16,8 @@ log = "0.4.14" thiserror = "1.0.30" wasmparser = "0.83.0" wit-bindgen-wasmer = { git = "https://github.com/wasmerio/wit-bindgen", branch = "wasmer" } +tracing = { version = "0.1.33", features = ["attributes"] } +tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } indexmap = { version = "1.8.0", features = ["serde-1"] } [dependencies.csv] diff --git a/crates/runtime/src/engine/zune.rs b/crates/runtime/src/engine/zune.rs index c275783c9d..a9043f2535 100644 --- a/crates/runtime/src/engine/zune.rs +++ b/crates/runtime/src/engine/zune.rs @@ -1,37 +1,36 @@ use std::{ + collections::{HashMap, HashSet}, convert::TryInto, fmt::{self, Display, Formatter}, - sync::{Arc, Mutex}, io::{Cursor, Read}, - collections::HashMap, + sync::{Arc, Mutex}, + borrow::Cow }; -use zip; -use anyhow::{Context, Error, anyhow}; +use anyhow::{anyhow, Context, Error}; +use hotg_rune_compiler::{diagnostics::Diagnostics, parse::yaml::*}; use hotg_rune_core::{ElementType as RuneElementType, Shape, TFLITE_MIMETYPE}; -use indexmap::IndexMap; -use wasmer::{ - Array, Function, Instance, LazyInit, Memory, Module, NativeFunc, - RuntimeError, Store, ValueType, WasmPtr, WasmerEnv, -}; - use hotg_runecoral::{ - AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, Tensor, - TensorDescriptor, TensorMut, + AccelerationBackend, + InferenceContext, + ElementType as RuneCoralElementType, + TensorDescriptor as RuneCoralTensorDescriptor, + Tensor as RuneCoralTensor, + TensorMut as RuneCoralTensorMut, }; - -use hotg_rune_compiler::{ - parse::yaml::*, - diagnostics::Diagnostics +use indexmap::IndexMap; +use wasmer::{ + Array, Function, ImportObject, Instance, LazyInit, Memory, Module, + NativeFunc, RuntimeError, Store, ValueType, WasmPtr, WasmerEnv, }; +use zip; +use self::{proc_block_v1::ProcBlockV1, runtime_v1::*}; use crate::{ callbacks::Callbacks, engine::{host_functions::HostFunctions, LoadError, WebAssemblyEngine}, }; -use self::{proc_block_v1::ProcBlockV1, runtime_v1::*}; - wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); @@ -39,214 +38,524 @@ wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); struct Runtime(Arc>); #[derive(Debug, Default)] -pub struct State { - pub arguments: HashMap, - pub inputs: HashMap, - pub outputs: HashMap, +struct State { + tensors: Vec>, } -pub struct ModelContext { - inference_context: InferenceContext +struct ModelNode { + context: InferenceContext, + input_tensors: HashSet, + output_tensors: HashSet, + state: Arc> } -pub struct ProcBlockContext { - instance: Instance +struct ProcBlockNode { + // input_tensors: HashMap, + // output_tensors: HashMap, + context: ProcBlockV1, + state: Arc>, } pub struct ZuneEngine { - tensors: Vec>, inputs: Vec, - input_tensors: HashMap, - output_tensors: HashMap, outputs: Vec, - model_contexts: HashMap, - procblock_contexts: HashMap, + models: HashMap, + procblocks: HashMap, pipeline: IndexMap, - processing_order: Vec - // resources + processing_order: Vec, + state: Arc>, // resources } impl WebAssemblyEngine for ZuneEngine { - fn load( - binary: &[u8], - _: Arc, - ) -> Result + fn load(binary: &[u8], _: Arc) -> Result where Self: Sized, { - let mut archive = zip::ZipArchive::new(Cursor::new(binary)).context("Unable to load Zune")?; - - let mut read_zip_resource_by_path = |path: &str| -> Result, LoadError> { - let mut requested_file = archive.by_name(path).context(anyhow!("Unable to find {} in zune", path))?; - let mut buffer = Vec::new(); - requested_file.read_to_end(&mut buffer).context(anyhow!("Unable to read {} from zune", path))?; - Ok(buffer) - }; - - let runefile = String::from_utf8(read_zip_resource_by_path("Runefile.yml")?).context("Unable to read Runefile")?; - let parsed_runefile = Document::parse(&runefile).context("Unable to parse Runefile")?; + let mut archive = zip::ZipArchive::new(Cursor::new(binary)) + .context("Unable to load Zune")?; + + let mut read_zip_resource_by_path = + |path: &str| -> Result, Error> { + let mut requested_file = + archive.by_name(path).with_context(|| { + anyhow!("Unable to find {} in zune", path) + })?; + let mut buffer = Vec::new(); + requested_file + .read_to_end(&mut buffer) + .with_context(|| anyhow!("Unable to read {} from zune", path))?; + Ok(buffer) + }; + + let runefile = + String::from_utf8(read_zip_resource_by_path("Runefile.yml")?) + .context("Unable to read Runefile")?; + let parsed_runefile = + Document::parse(&runefile).context("Unable to parse Runefile")?; let pipeline = &parsed_runefile.to_v1().pipeline; - let mut model_contexts: HashMap = HashMap::new(); - let mut procblock_contexts: HashMap = HashMap::new(); - let mut inputs = Vec::new(); - let mut outputs = Vec::new(); - - for item in pipeline { - // Collect each output tensor into tensors - let stage_name = item.0; - match item.1 { - Stage::Capability(_) => { - inputs.push(stage_name.to_string()); - }, - Stage::Model(stage) => { - // Instantiating the model's inference context here because that way model_data gets deallocated once we are done with it - // This way memory usage is under control - let model_data = read_zip_resource_by_path(&stage.model.to_string()) - .context(format!("Unable to read model from zune {}", stage.model))?; - let inference_context = - InferenceContext::create_context(TFLITE_MIMETYPE, &model_data, AccelerationBackend::NONE) - .context(format!("Error Instantiating model from zune {}", stage.model))?; - - model_contexts.insert(stage_name.to_string(), ModelContext { inference_context }); - }, - Stage::ProcBlock(stage) => { - println!("Pipeline stage: {} proc block {:?} {}", stage_name, stage.args, stage.proc_block.base ); - let wasm = read_zip_resource_by_path(&stage.proc_block.base).context("Unable to load the proc_block")?; - }, - Stage::Out(_) => { - outputs.push(stage_name.to_string()); - } - } - } - - let (tensors, input_tensors, output_tensors, processing_order) - = get_tensors(&inputs, &outputs, &pipeline).context(anyhow!("Unable to map out input/output tensors"))?; + let inputs: Vec<_> = pipeline + .iter() + .filter_map(|(k, v)| match v { + Stage::Capability(_) => Some(k.clone()), + _ => None, + }) + .collect(); + + let outputs: Vec<_> = pipeline + .iter() + .filter_map(|(k, v)| match v { + Stage::Out(_) => Some(k.clone()), + _ => None, + }) + .collect(); + + let (tensors, input_tensors, output_tensors, processing_order) = + get_tensors(&inputs, &outputs, &pipeline) + .context(anyhow!("Unable to map out input/output tensors"))?; + + let state = Arc::new(Mutex::new(State { tensors })); + + let (model_contexts, procblock_contexts) = instantiate_nodes( + pipeline, + read_zip_resource_by_path, + &state, + input_tensors, + output_tensors, + ) + .map_err(LoadError::Other)?; - /* - println!("input_tensors: {:?}", &input_tensors); - println!("output_tensors: {:?}", &output_tensors); - println!("processing_order: {:?}", &processing_order); - */ + // println!("input_tensors: {:?}", &input_tensors); + // println!("output_tensors: {:?}", &output_tensors); + // println!("processing_order: {:?}", &processing_order); Ok(ZuneEngine { - tensors, inputs, - input_tensors, outputs, - output_tensors, - model_contexts, - procblock_contexts, + models: model_contexts, + procblocks: procblock_contexts, pipeline: pipeline.to_owned(), - processing_order + processing_order, + state, }) } fn init(&mut self) -> Result<(), Error> { - //TODO: Call each proc block's graph() and each model's inputs/outputs to instantiate the tensors + // TODO: Call each proc block's graph() and each model's inputs/outputs + // to allocate the tensors with correct dimensions Ok(()) } fn predict(&mut self) -> Result<(), Error> { + for stage_name in &self.processing_order { + let stage = self.pipeline.get(stage_name).unwrap(); + match stage { + Stage::Model(stage) => { + self.models.get_mut(stage_name).unwrap().run()?; + }, + Stage::ProcBlock(stage) => { + self.procblocks.get_mut(stage_name).unwrap().run()?; + }, + _ => {}, + } + } Ok(()) } } -fn get_tensors(inputs: &Vec, outputs: &Vec, pipeline: &IndexMap) - -> Result<(Vec>, HashMap, HashMap, Vec), Error> { - let mut nodes_to_visit = outputs.clone(); - let mut nodes_visited = Vec::new(); - let mut tensors: Vec> = Vec::new(); - let mut output_tensors: HashMap = HashMap::new(); - let mut input_tensors: HashMap = HashMap::new(); +impl ModelNode { + fn load( + node_id: &str, + node_data: &ModelStage, + model_data: &[u8], + state: Arc>, + input_tensors: &HashMap, + output_tensors: &HashMap, + ) -> Result { + // Create Inference Context + let context = InferenceContext::create_context( + TFLITE_MIMETYPE, + &model_data, + AccelerationBackend::NONE, + ) + .with_context(|| { + format!( + "Error Instantiating model from zune for stage: {}", + &node_id + ) + })?; + + let tensor_from_descriptor = |t: &RuneCoralTensorDescriptor| -> TensorResult { + let element_type = get_element_type(t); + let dimensions = t.shape.iter().map(|&x| x as u32).collect(); + let buffer_size = get_buffer_size(element_type, &dimensions); + + TensorResult { + element_type, + dimensions, + buffer: vec![0; buffer_size], + } + }; - let key = |node_name: &str, tensor_index: Option| format!("{}.{}", node_name, tensor_index.or(Some(0)).unwrap()); + // Returns the list of tensor indices in the State's tensors + let allocate_tensors = |tensor_type: &str, model_tensors: &mut Iterator, pipeline_tensors: &HashMap| -> Result, Error> { + let mut result: HashSet = HashSet::new(); + let mut i = 0; + let mut s = state.lock().unwrap(); + + while let Some(model_tensor) = model_tensors.next() { + let model_tensor = tensor_from_descriptor(&model_tensor); + let tensor_key = key(&node_id, Some(i)); + let tensor_id = *pipeline_tensors.get(&tensor_key) + .ok_or_else(|| anyhow!("Unable to find pipeline_tensor for {} tensor with key {}", &tensor_type, &tensor_key))?; + + if s.tensors[tensor_id].is_none() { + s.tensors[tensor_id] = Some(model_tensor); + } else { + let state_tensor = s.tensors[tensor_id].as_ref().unwrap(); + if state_tensor.dimensions != model_tensor.dimensions || state_tensor.element_type != model_tensor.element_type { + return Err(anyhow!("Pipeline tensor for {} with key {} doesn't match model tensor", &tensor_type, &tensor_key)); + } + } + result.insert(tensor_id); - // For Inputs/Capabilities - input tensors and output tensors are the same? - for item in inputs { - tensors.push(None); - input_tensors.insert(key(item, Some(0)), tensors.len() - 1); - output_tensors.insert(key(item, Some(0)), tensors.len() - 1); - } + i += 1; + } - // Do a depth first traversal of the tree structure to determine the order of processing/calling predict() - // Also allocate the output tensors of each node along the way - while !nodes_to_visit.is_empty() { - let node = nodes_to_visit.pop().unwrap(); - nodes_visited.push(node.clone()); + Ok(result) + }; - let stage = pipeline.get(&node).unwrap(); - for output_index in 0..stage.output_types().len() { - tensors.push(None); - output_tensors.insert(key(&node, Some(output_index)), tensors.len() - 1); - } + let input_tensors = allocate_tensors("input", &mut context.inputs(), &input_tensors)?; + let output_tensors = allocate_tensors("output", &mut context.outputs(), &output_tensors)?; + + Ok(ModelNode { context, input_tensors, output_tensors, state: state.clone() }) + } + + fn run(&mut self) -> Result<(), Error> { + // We are recreating the input_tensors and output_tensors every time before predict + // because wasm linear memory might have changed the locations + // TODO: There's an optimization that can happen here.. but just not yet + let mut inputs: Vec = Vec::new(); + let mut outputs: Vec = Vec::new(); + let mut state = self.state.lock().unwrap(); - for input in stage.inputs() { - if !nodes_visited.contains(&input.name) { - nodes_to_visit.push(input.name.clone()); + state + .tensors + .iter_mut() + .enumerate() + .for_each(|(i, t)| { + let mut pipeline_tensor = t.as_mut().unwrap(); + if self.input_tensors.contains(&i) { + unsafe { + inputs.push(RuneCoralTensor { + element_type: get_runecoral_element_type(&pipeline_tensor.element_type), + shape: Cow::Borrowed(std::slice::from_raw_parts(pipeline_tensor.dimensions.as_ptr() as *const i32, pipeline_tensor.dimensions.len())), + buffer: &pipeline_tensor.buffer + }) + } + } else if self.output_tensors.contains(&i) { + unsafe { + outputs.push(RuneCoralTensorMut { + element_type: get_runecoral_element_type(&pipeline_tensor.element_type), + shape: Cow::Borrowed(std::slice::from_raw_parts(pipeline_tensor.dimensions.as_ptr() as *const i32, pipeline_tensor.dimensions.len())), + buffer: &mut pipeline_tensor.buffer + }) + } + } else { + // Do nothing } - } + }); + + + self.context.infer(&inputs, &mut outputs).map_err(|e| anyhow!(e.to_string())) + } +} + +impl ProcBlockNode { + fn load( + node_id: &str, + node_data: &ProcBlockStage, + wasm: &[u8], + store: &Store, + mut imports: &mut ImportObject, + state: Arc>, + input_tensors: &HashMap, + output_tensors: &HashMap, + ) -> Result { + let module = + Module::new(&store, wasm).context("Unable to load the module")?; + + let (pb, _) = ProcBlockV1::instantiate(&store, &module, &mut imports) + .context("Unable to instantiate the WebAssembly module")?; + + Ok(ProcBlockNode { + context: pb, + state: state.clone() + }) + } + + fn run(&mut self) -> Result<(), Error> { + todo!() + } +} + +fn get_buffer_size(element_type: ElementType, dimensions: &Vec) -> usize { + ((dimensions.iter().fold(1, |a, &b| a * b) * get_bytes_per_element(element_type))) as usize +} + +fn get_bytes_per_element(element_type: ElementType) -> u32 { + match element_type { + ElementType::I16 => 2, + ElementType::I32 | ElementType::F32 => 4, + ElementType::I64 | ElementType::F64 => 8, + _ => 1, + } +} + +fn get_element_type(t: &RuneCoralTensorDescriptor) -> ElementType { + match t.element_type { + RuneCoralElementType::UInt8 => ElementType::U8, + RuneCoralElementType::Int8 => ElementType::I8, + RuneCoralElementType::Int16 => ElementType::I16, + RuneCoralElementType::Int32 => ElementType::I32, + RuneCoralElementType::Float32 => ElementType::F32, + RuneCoralElementType::Int64 => ElementType::I64, + RuneCoralElementType::Float64 => ElementType::F64, + RuneCoralElementType::String => ElementType::Utf8, + // TODO: Implement support for all the element types + _ => ElementType::U8, + } +} + +fn get_runecoral_element_type(t: &ElementType) -> RuneCoralElementType { + match t { + ElementType::U8 => RuneCoralElementType::UInt8, + ElementType::I8 => RuneCoralElementType::Int8, + ElementType::I16 => RuneCoralElementType::Int16, + ElementType::I32 => RuneCoralElementType::Int32, + ElementType::F32 => RuneCoralElementType::Float32, + ElementType::I64 => RuneCoralElementType::Int64, + ElementType::F64 => RuneCoralElementType::Float64, + ElementType::Utf8 => RuneCoralElementType::String, + // TODO: Implement support for all the element types + _ => RuneCoralElementType::NoType, + } +} + +fn instantiate_nodes( + pipeline: &IndexMap, + mut read_zip_resource_by_path: impl FnMut(&str) -> Result, Error>, + state: &Arc>, + input_tensors: HashMap, + output_tensors: HashMap, +) -> Result<(HashMap, HashMap), Error> +{ + let mut models: HashMap = HashMap::new(); + let mut procblocks: HashMap = HashMap::new(); + + let store = Store::default(); + let mut imports = ImportObject::default(); + let mut runtime = Runtime(state.clone()); + add_to_imports(&store, &mut imports, runtime.clone()); + + for item in pipeline { + // Collect each output tensor into tensors + let stage_name = item.0; + match item.1 { + Stage::Capability(_) => { + // inputs.push(stage_name.to_string()); + }, + Stage::Model(stage) => { + // Instantiating the model's inference context here because that + // way model_data gets deallocated once we are done with it + // This way memory usage is under control + let model_data = + read_zip_resource_by_path(&stage.model.to_string()) + .with_context(|| anyhow!( + "Unable to read model from zune {}", + stage.model + ))?; + + models.insert( + stage_name.to_string(), + ModelNode::load( + &stage_name, + &stage, + &model_data, + runtime.0.clone(), + &input_tensors, + &output_tensors, + )?, + ); + }, + Stage::ProcBlock(stage) => { + println!( + "Pipeline stage: {} proc block {:?} {}", + stage_name, stage.args, stage.proc_block.base + ); + let wasm = read_zip_resource_by_path(&stage.proc_block.base) + .context("Unable to load the proc_block")?; + + procblocks.insert( + stage_name.to_string(), + ProcBlockNode::load( + &stage_name, + &stage, + &wasm, + &store, + &mut imports, + runtime.0.clone(), + &input_tensors, + &output_tensors, + )?, + ); + }, + Stage::Out(_) => { + // outputs.push(stage_name.to_string()); + }, } + } - // For each stage in the pipeline, since the inputs have to come from the outputs of other stages, simply map to the same tensor - for item in pipeline { - // Collect each output tensor into tensors - let stage_name = item.0; - for i in 0..item.1.inputs().len() { - let input = &item.1.inputs()[i]; - let input_key = key(&input.name, input.index); - let &input_tensor_index = output_tensors.get(&input_key).context(anyhow!("Invalid input key specified: {}", &input_key))?; - input_tensors.insert(key(stage_name, Some(i)), input_tensor_index); + Ok((models, procblocks)) +} + +fn get_tensors( + inputs: &Vec, + outputs: &Vec, + pipeline: &IndexMap, +) -> Result< + ( + Vec>, + HashMap, + HashMap, + Vec, + ), + Error, +> { + let mut nodes_to_visit = outputs.clone(); + let mut nodes_visited = Vec::new(); + let mut tensors: Vec> = Vec::new(); + let mut output_tensors: HashMap = HashMap::new(); + let mut input_tensors: HashMap = HashMap::new(); + + // For Inputs/Capabilities - input tensors and output tensors are the same? + for item in inputs { + tensors.push(None); + input_tensors.insert(key(item, Some(0)), tensors.len() - 1); + output_tensors.insert(key(item, Some(0)), tensors.len() - 1); + } + + // Do a depth first traversal of the tree structure to determine the order + // of processing/calling predict() Also allocate the output tensors of + // each node along the way + while !nodes_to_visit.is_empty() { + let node = nodes_to_visit.pop().unwrap(); + nodes_visited.push(node.clone()); + + let stage = pipeline.get(&node).unwrap(); + for output_index in 0..stage.output_types().len() { + tensors.push(None); + output_tensors + .insert(key(&node, Some(output_index)), tensors.len() - 1); + } + + for input in stage.inputs() { + if !nodes_visited.contains(&input.name) { + nodes_to_visit.push(input.name.clone()); } } + } + + // For each stage in the pipeline, since the inputs have to come from the + // outputs of other stages, simply map to the same tensor + for item in pipeline { + // Collect each output tensor into tensors + let stage_name = item.0; + for i in 0..item.1.inputs().len() { + let input = &item.1.inputs()[i]; + let input_key = key(&input.name, input.index); + let &input_tensor_index = output_tensors.get(&input_key).context( + anyhow!("Invalid input key specified: {}", &input_key), + )?; + input_tensors.insert(key(stage_name, Some(i)), input_tensor_index); + } + } + + nodes_visited.reverse(); - nodes_visited.reverse(); + Ok((tensors, input_tensors, output_tensors, nodes_visited)) +} - Ok((tensors, input_tensors, output_tensors, nodes_visited)) +fn key(node_name: &str, tensor_index: Option) -> String { + format!("{}.{}", node_name, tensor_index.or(Some(0)).unwrap()) } -/* #[derive(Debug, Clone)] pub enum Never {} -impl runtime_v1::RuntimeV1 for ZuneEngine { +impl runtime_v1::RuntimeV1 for Runtime { type ArgumentHint = Never; type ArgumentMetadata = Never; type GraphContext = Never; type KernelContext = Arc>; type Metadata = Never; + type Model = Never; type TensorHint = Never; type TensorMetadata = Never; - type LogMetadata = Never; fn metadata_new(&mut self, _name: &str, _version: &str) -> Self::Metadata { todo!() } - fn metadata_set_description(&mut self, _self_: &Self::Metadata, _description: &str) { + + fn metadata_set_description( + &mut self, + _self_: &Self::Metadata, + _description: &str, + ) { todo!() } + fn metadata_set_repository(&mut self, _self_: &Self::Metadata, _url: &str) { todo!() } + fn metadata_set_homepage(&mut self, _self_: &Self::Metadata, _url: &str) { todo!() } + fn metadata_add_tag(&mut self, _self_: &Self::Metadata, _tag: &str) { todo!() } - fn metadata_add_argument(&mut self, _self_: &Self::Metadata, _arg: &Self::ArgumentMetadata) { + + fn metadata_add_argument( + &mut self, + _self_: &Self::Metadata, + _arg: &Self::ArgumentMetadata, + ) { todo!() } - fn metadata_add_input(&mut self, _self_: &Self::Metadata, _metadata: &Self::TensorMetadata) { + + fn metadata_add_input( + &mut self, + _self_: &Self::Metadata, + _metadata: &Self::TensorMetadata, + ) { todo!() } - fn metadata_add_output(&mut self, _self_: &Self::Metadata, _metadata: &Self::TensorMetadata) { + + fn metadata_add_output( + &mut self, + _self_: &Self::Metadata, + _metadata: &Self::TensorMetadata, + ) { todo!() } + fn argument_metadata_new(&mut self, _name: &str) -> Self::ArgumentMetadata { todo!() } + fn argument_metadata_set_description( &mut self, _self_: &Self::ArgumentMetadata, @@ -254,6 +563,7 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { ) { todo!() } + fn argument_metadata_set_default_value( &mut self, _self_: &Self::ArgumentMetadata, @@ -261,6 +571,7 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { ) { todo!() } + fn argument_metadata_add_hint( &mut self, _self_: &Self::ArgumentMetadata, @@ -268,9 +579,11 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { ) { todo!() } + fn tensor_metadata_new(&mut self, _name: &str) -> Self::TensorMetadata { todo!() } + fn tensor_metadata_set_description( &mut self, _self_: &Self::TensorMetadata, @@ -278,6 +591,7 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { ) { todo!() } + fn tensor_metadata_add_hint( &mut self, _self_: &Self::TensorMetadata, @@ -285,37 +599,52 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { ) { todo!() } - fn interpret_as_image(&mut self) -> Self::TensorHint { - todo!() - } - fn interpret_as_audio(&mut self) -> Self::TensorHint { - todo!() - } + + fn interpret_as_image(&mut self) -> Self::TensorHint { todo!() } + + fn interpret_as_audio(&mut self) -> Self::TensorHint { todo!() } + fn supported_shapes( &mut self, _supported_element_types: Vec, - _dimensions: Dimensions<'_>, + _dimensions: DimensionsParam<'_>, ) -> Self::TensorHint { todo!() } - fn interpret_as_number_in_range(&mut self, _min: &str, _max: &str) -> Self::ArgumentHint { - todo!() - } - fn interpret_as_string_in_enum(&mut self, _string_enum: Vec<&str>) -> Self::ArgumentHint { - todo!() - } - fn non_negative_number(&mut self) -> Self::ArgumentHint { + + fn interpret_as_number_in_range( + &mut self, + _min: &str, + _max: &str, + ) -> Self::ArgumentHint { todo!() } - fn supported_argument_type(&mut self, _hint: ArgumentType) -> Self::ArgumentHint { + + fn interpret_as_string_in_enum( + &mut self, + _string_enum: Vec<&str>, + ) -> Self::ArgumentHint { todo!() } - fn register_node(&mut self, _metadata: &Self::Metadata) { + + fn non_negative_number(&mut self) -> Self::ArgumentHint { todo!() } + + fn supported_argument_type( + &mut self, + _hint: ArgumentType, + ) -> Self::ArgumentHint { todo!() } - fn graph_context_for_node(&mut self, _node_id: &str) -> Option { + + fn register_node(&mut self, _metadata: &Self::Metadata) { todo!() } + + fn graph_context_for_node( + &mut self, + _node_id: &str, + ) -> Option { todo!() } + fn graph_context_get_argument( &mut self, _self_: &Self::GraphContext, @@ -323,41 +652,50 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { ) -> Option { todo!() } + fn graph_context_add_input_tensor( &mut self, _self_: &Self::GraphContext, _name: &str, _element_type: ElementType, - _dimensions: Dimensions<'_>, + _dimensions: DimensionsParam<'_>, ) { todo!() } + fn graph_context_add_output_tensor( &mut self, _self_: &Self::GraphContext, _name: &str, _element_type: ElementType, - _dimensions: Dimensions<'_>, + _dimensions: DimensionsParam<'_>, ) { todo!() } - fn kernel_context_for_node(&mut self, _node_id: &str) -> Option { + + fn kernel_context_for_node( + &mut self, + _node_id: &str, + ) -> Option { Some(self.0.clone()) } + fn kernel_context_get_argument( &mut self, state: &Arc>, name: &str, ) -> Option { - state.lock().unwrap().arguments.get(name).cloned() + todo!() } + fn kernel_context_get_input_tensor( &mut self, state: &Arc>, name: &str, ) -> Option { - state.lock().unwrap().inputs.get(name).cloned() + todo!() } + fn kernel_context_set_output_tensor( &mut self, state: &Arc>, @@ -368,23 +706,17 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { dimensions, }: TensorParam<'_>, ) { - let tensor = TensorResult { - element_type, - dimensions: dimensions.iter().map(|d| d.get()).collect(), - buffer: buffer.to_vec(), - }; - state - .lock() - .unwrap() - .outputs - .insert(name.to_string(), tensor); + todo!() } - fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { - true - } + fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { true } - fn log(&mut self, metadata: LogMetadata, message: &str, data: Vec<(&'_ str, LogValue<'_>)>) { + fn log( + &mut self, + metadata: LogMetadata, + message: &str, + data: Vec<(&'_ str, LogValue<'_>)>, + ) { let level = match metadata.level { LogLevel::Trace => tracing::Level::TRACE, LogLevel::Debug => tracing::Level::DEBUG, @@ -414,5 +746,46 @@ impl runtime_v1::RuntimeV1 for ZuneEngine { message, ); } + + fn kernel_context_get_global_input( + &mut self, + self_: &Self::KernelContext, + name: &str, + ) -> Option { + todo!() + } + + fn kernel_context_set_global_output( + &mut self, + self_: &Self::KernelContext, + name: &str, + tensor: TensorParam<'_>, + ) { + todo!() + } + + fn model_load( + &mut self, + model_format: &str, + model: &[u8], + arguments: Vec<(&str, &str)>, + ) -> Result { + todo!() + } + + fn model_inputs(&mut self, self_: &Self::Model) -> Vec { + todo!() + } + + fn model_outputs(&mut self, self_: &Self::Model) -> Vec { + todo!() + } + + fn model_infer( + &mut self, + self_: &Self::Model, + inputs: Vec>, + ) -> Result, ModelInferError> { + todo!() + } } -*/ \ No newline at end of file From 34251293d865d97e2d775550dcfbd2176654b7bc Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 17 May 2022 06:44:43 +0530 Subject: [PATCH 46/84] Zune: Implement graph_context_add_*_tensors --- crates/runtime/src/engine/zune.rs | 344 +++++++++++++++++++++--------- 1 file changed, 243 insertions(+), 101 deletions(-) diff --git a/crates/runtime/src/engine/zune.rs b/crates/runtime/src/engine/zune.rs index a9043f2535..0a470edc17 100644 --- a/crates/runtime/src/engine/zune.rs +++ b/crates/runtime/src/engine/zune.rs @@ -1,21 +1,18 @@ use std::{ + borrow::Cow, collections::{HashMap, HashSet}, convert::TryInto, fmt::{self, Display, Formatter}, io::{Cursor, Read}, sync::{Arc, Mutex}, - borrow::Cow }; use anyhow::{anyhow, Context, Error}; use hotg_rune_compiler::{diagnostics::Diagnostics, parse::yaml::*}; use hotg_rune_core::{ElementType as RuneElementType, Shape, TFLITE_MIMETYPE}; use hotg_runecoral::{ - AccelerationBackend, - InferenceContext, - ElementType as RuneCoralElementType, - TensorDescriptor as RuneCoralTensorDescriptor, - Tensor as RuneCoralTensor, + AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, + Tensor as RuneCoralTensor, TensorDescriptor as RuneCoralTensorDescriptor, TensorMut as RuneCoralTensorMut, }; use indexmap::IndexMap; @@ -35,25 +32,29 @@ wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); #[derive(Debug, Default, Clone, wasmer::WasmerEnv)] -struct Runtime(Arc>); +struct Runtime { + shared_state: Arc>, +} #[derive(Debug, Default)] struct State { tensors: Vec>, + graph_contexts: HashMap, + tensor_constraints: Vec> } struct ModelNode { context: InferenceContext, input_tensors: HashSet, output_tensors: HashSet, - state: Arc> + shared_state: Arc>, } struct ProcBlockNode { - // input_tensors: HashMap, - // output_tensors: HashMap, + input_tensors: HashMap, + output_tensors: HashMap, context: ProcBlockV1, - state: Arc>, + shared_state: Arc> } pub struct ZuneEngine { @@ -63,7 +64,7 @@ pub struct ZuneEngine { procblocks: HashMap, pipeline: IndexMap, processing_order: Vec, - state: Arc>, // resources + shared_state: Arc>, // resources } impl WebAssemblyEngine for ZuneEngine { @@ -81,9 +82,9 @@ impl WebAssemblyEngine for ZuneEngine { anyhow!("Unable to find {} in zune", path) })?; let mut buffer = Vec::new(); - requested_file - .read_to_end(&mut buffer) - .with_context(|| anyhow!("Unable to read {} from zune", path))?; + requested_file.read_to_end(&mut buffer).with_context(|| { + anyhow!("Unable to read {} from zune", path) + })?; Ok(buffer) }; @@ -114,21 +115,30 @@ impl WebAssemblyEngine for ZuneEngine { get_tensors(&inputs, &outputs, &pipeline) .context(anyhow!("Unable to map out input/output tensors"))?; - let state = Arc::new(Mutex::new(State { tensors })); + let graph_contexts = pipeline + .iter() + .map(|(k, v)| { + + let arguments = + v.args() + .iter() + .map(|(name, argument)| (name.clone(), argument.to_string())) + .collect(); + (k.clone(), GraphContext{ arguments, input_tensors: Vec::new(), output_tensors: Vec::new() }) + }) + .collect(); + + let shared_state = Arc::new(Mutex::new(State { tensors, graph_contexts, tensor_constraints: Vec::new() })); let (model_contexts, procblock_contexts) = instantiate_nodes( pipeline, read_zip_resource_by_path, - &state, + &shared_state, input_tensors, output_tensors, ) .map_err(LoadError::Other)?; - // println!("input_tensors: {:?}", &input_tensors); - // println!("output_tensors: {:?}", &output_tensors); - // println!("processing_order: {:?}", &processing_order); - Ok(ZuneEngine { inputs, outputs, @@ -136,7 +146,7 @@ impl WebAssemblyEngine for ZuneEngine { procblocks: procblock_contexts, pipeline: pipeline.to_owned(), processing_order, - state, + shared_state, }) } @@ -169,7 +179,7 @@ impl ModelNode { node_id: &str, node_data: &ModelStage, model_data: &[u8], - state: Arc>, + shared_state: &Arc>, input_tensors: &HashMap, output_tensors: &HashMap, ) -> Result { @@ -186,38 +196,61 @@ impl ModelNode { ) })?; - let tensor_from_descriptor = |t: &RuneCoralTensorDescriptor| -> TensorResult { - let element_type = get_element_type(t); - let dimensions = t.shape.iter().map(|&x| x as u32).collect(); - let buffer_size = get_buffer_size(element_type, &dimensions); + let tensor_from_descriptor = + |t: &RuneCoralTensorDescriptor| -> TensorResult { + let element_type = get_element_type(t); + let dimensions = t.shape.iter().map(|&x| x as u32).collect(); + let buffer_size = get_buffer_size(element_type, &dimensions); - TensorResult { - element_type, - dimensions, - buffer: vec![0; buffer_size], - } - }; + TensorResult { + element_type, + dimensions, + buffer: vec![0; buffer_size], + } + }; // Returns the list of tensor indices in the State's tensors - let allocate_tensors = |tensor_type: &str, model_tensors: &mut Iterator, pipeline_tensors: &HashMap| -> Result, Error> { + let allocate_tensors = |tensor_type: &str, + model_tensors: &mut dyn Iterator< + Item = RuneCoralTensorDescriptor, + >, + pipeline_tensors: &HashMap| + -> Result, Error> { let mut result: HashSet = HashSet::new(); let mut i = 0; - let mut s = state.lock().unwrap(); + let mut s = shared_state.lock().unwrap(); while let Some(model_tensor) = model_tensors.next() { let model_tensor = tensor_from_descriptor(&model_tensor); let tensor_key = key(&node_id, Some(i)); - let tensor_id = *pipeline_tensors.get(&tensor_key) - .ok_or_else(|| anyhow!("Unable to find pipeline_tensor for {} tensor with key {}", &tensor_type, &tensor_key))?; - - if s.tensors[tensor_id].is_none() { - s.tensors[tensor_id] = Some(model_tensor); - } else { - let state_tensor = s.tensors[tensor_id].as_ref().unwrap(); - if state_tensor.dimensions != model_tensor.dimensions || state_tensor.element_type != model_tensor.element_type { - return Err(anyhow!("Pipeline tensor for {} with key {} doesn't match model tensor", &tensor_type, &tensor_key)); - } + let tensor_id = + *pipeline_tensors.get(&tensor_key).ok_or_else(|| { + anyhow!( + "Unable to find pipeline_tensor for {} tensor \ + with key {}", + &tensor_type, + &tensor_key + ) + })?; + + match s.tensors[tensor_id] { + Some(ref t) + if t.dimensions != model_tensor.dimensions + || t.element_type != model_tensor.element_type => + { + return Err(anyhow!( + "Pipeline tensor for {} with key {} doesn't match \ + model tensor", + &tensor_type, + &tensor_key + )) + }, + Some(_) => {}, + ref mut other => { + other.insert(model_tensor); + }, } + result.insert(tensor_id); i += 1; @@ -226,49 +259,69 @@ impl ModelNode { Ok(result) }; - let input_tensors = allocate_tensors("input", &mut context.inputs(), &input_tensors)?; - let output_tensors = allocate_tensors("output", &mut context.outputs(), &output_tensors)?; + let input_tensors = + allocate_tensors("input", &mut context.inputs(), &input_tensors)?; + let output_tensors = allocate_tensors( + "output", + &mut context.outputs(), + &output_tensors, + )?; - Ok(ModelNode { context, input_tensors, output_tensors, state: state.clone() }) + Ok(ModelNode { + context, + input_tensors, + output_tensors, + shared_state: shared_state.clone(), + }) } fn run(&mut self) -> Result<(), Error> { - // We are recreating the input_tensors and output_tensors every time before predict - // because wasm linear memory might have changed the locations - // TODO: There's an optimization that can happen here.. but just not yet + // We are recreating the input_tensors and output_tensors every time + // before predict because wasm linear memory might have changed + // the locations TODO: There's an optimization that can happen + // here.. but just not yet let mut inputs: Vec = Vec::new(); let mut outputs: Vec = Vec::new(); - let mut state = self.state.lock().unwrap(); + let mut state = self.shared_state.lock().unwrap(); - state - .tensors - .iter_mut() - .enumerate() - .for_each(|(i, t)| { + state.tensors.iter_mut().enumerate().for_each(|(i, t)| { + if self.input_tensors.contains(&i) { let mut pipeline_tensor = t.as_mut().unwrap(); - if self.input_tensors.contains(&i) { - unsafe { - inputs.push(RuneCoralTensor { - element_type: get_runecoral_element_type(&pipeline_tensor.element_type), - shape: Cow::Borrowed(std::slice::from_raw_parts(pipeline_tensor.dimensions.as_ptr() as *const i32, pipeline_tensor.dimensions.len())), - buffer: &pipeline_tensor.buffer - }) - } - } else if self.output_tensors.contains(&i) { - unsafe { - outputs.push(RuneCoralTensorMut { - element_type: get_runecoral_element_type(&pipeline_tensor.element_type), - shape: Cow::Borrowed(std::slice::from_raw_parts(pipeline_tensor.dimensions.as_ptr() as *const i32, pipeline_tensor.dimensions.len())), - buffer: &mut pipeline_tensor.buffer - }) - } - } else { - // Do nothing + unsafe { + inputs.push(RuneCoralTensor { + element_type: get_runecoral_element_type( + &pipeline_tensor.element_type, + ), + shape: Cow::Borrowed(std::slice::from_raw_parts( + pipeline_tensor.dimensions.as_ptr() as *const i32, + pipeline_tensor.dimensions.len(), + )), + buffer: &pipeline_tensor.buffer, + }) } - }); + } else if self.output_tensors.contains(&i) { + let mut pipeline_tensor = t.as_mut().unwrap(); + unsafe { + outputs.push(RuneCoralTensorMut { + element_type: get_runecoral_element_type( + &pipeline_tensor.element_type, + ), + shape: Cow::Borrowed(std::slice::from_raw_parts( + pipeline_tensor.dimensions.as_ptr() as *const i32, + pipeline_tensor.dimensions.len(), + )), + buffer: &mut pipeline_tensor.buffer, + }) + } + } else { + // Do nothing + } + }); - self.context.infer(&inputs, &mut outputs).map_err(|e| anyhow!(e.to_string())) + self.context + .infer(&inputs, &mut outputs) + .map_err(|e| anyhow!(e.to_string())) } } @@ -279,29 +332,33 @@ impl ProcBlockNode { wasm: &[u8], store: &Store, mut imports: &mut ImportObject, - state: Arc>, + shared_state: &Arc>, input_tensors: &HashMap, output_tensors: &HashMap, ) -> Result { let module = Module::new(&store, wasm).context("Unable to load the module")?; - let (pb, _) = ProcBlockV1::instantiate(&store, &module, &mut imports) - .context("Unable to instantiate the WebAssembly module")?; + let (pb, _) = + ProcBlockV1::instantiate(&store, &module, &mut imports) + .context("Unable to instantiate the WebAssembly module")?; + + let result = pb.graph(node_id); Ok(ProcBlockNode { + input_tensors: HashMap::new(), + output_tensors: HashMap::new(), context: pb, - state: state.clone() + shared_state: shared_state.clone(), }) } - fn run(&mut self) -> Result<(), Error> { - todo!() - } + fn run(&mut self) -> Result<(), Error> { Ok(()) } } fn get_buffer_size(element_type: ElementType, dimensions: &Vec) -> usize { - ((dimensions.iter().fold(1, |a, &b| a * b) * get_bytes_per_element(element_type))) as usize + (dimensions.iter().fold(1, |a, &b| a * b) + * get_bytes_per_element(element_type)) as usize } fn get_bytes_per_element(element_type: ElementType) -> u32 { @@ -346,7 +403,7 @@ fn get_runecoral_element_type(t: &ElementType) -> RuneCoralElementType { fn instantiate_nodes( pipeline: &IndexMap, mut read_zip_resource_by_path: impl FnMut(&str) -> Result, Error>, - state: &Arc>, + shared_state: &Arc>, input_tensors: HashMap, output_tensors: HashMap, ) -> Result<(HashMap, HashMap), Error> @@ -356,7 +413,7 @@ fn instantiate_nodes( let store = Store::default(); let mut imports = ImportObject::default(); - let mut runtime = Runtime(state.clone()); + let mut runtime = Runtime{ shared_state: shared_state.clone() }; add_to_imports(&store, &mut imports, runtime.clone()); for item in pipeline { @@ -372,10 +429,12 @@ fn instantiate_nodes( // This way memory usage is under control let model_data = read_zip_resource_by_path(&stage.model.to_string()) - .with_context(|| anyhow!( - "Unable to read model from zune {}", - stage.model - ))?; + .with_context(|| { + anyhow!( + "Unable to read model from zune {}", + stage.model + ) + })?; models.insert( stage_name.to_string(), @@ -383,7 +442,7 @@ fn instantiate_nodes( &stage_name, &stage, &model_data, - runtime.0.clone(), + &shared_state, &input_tensors, &output_tensors, )?, @@ -405,7 +464,7 @@ fn instantiate_nodes( &wasm, &store, &mut imports, - runtime.0.clone(), + &shared_state, &input_tensors, &output_tensors, )?, @@ -494,15 +553,58 @@ fn key(node_name: &str, tensor_index: Option) -> String { #[derive(Debug, Clone)] pub enum Never {} +#[derive(Debug, Clone)] +struct Metadata { + description: String, + repository: String, + homepage: String, + tags: Vec, + arguments: Vec, + inputs: Vec, + outputs: Vec +} + +#[derive(Debug, Clone)] +struct ArgumentMetadata { + description: String, + default_value: String, + // hint: Vec +} + +#[derive(Debug, Clone)] +struct TensorMetadata { + +} + +#[derive(Debug, Clone)] +enum Dimensions { + Dynamic, + Fixed(Vec) +} + +#[derive(Debug, Clone)] +struct TensorConstraint { + name: String, + element_type: ElementType, + dimensions: Dimensions +} + +#[derive(Debug, Default, Clone)] +struct GraphContext { + arguments: HashMap, + input_tensors: Vec, + output_tensors: Vec +} + impl runtime_v1::RuntimeV1 for Runtime { type ArgumentHint = Never; type ArgumentMetadata = Never; - type GraphContext = Never; type KernelContext = Arc>; - type Metadata = Never; + type Metadata = Metadata; type Model = Never; type TensorHint = Never; - type TensorMetadata = Never; + type TensorMetadata = TensorMetadata; + type GraphContext = String; fn metadata_new(&mut self, _name: &str, _version: &str) -> Self::Metadata { todo!() @@ -642,7 +744,12 @@ impl runtime_v1::RuntimeV1 for Runtime { &mut self, _node_id: &str, ) -> Option { - todo!() + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(_node_id) + .and_then(|_| Some(_node_id.to_string())) } fn graph_context_get_argument( @@ -650,7 +757,12 @@ impl runtime_v1::RuntimeV1 for Runtime { _self_: &Self::GraphContext, _name: &str, ) -> Option { - todo!() + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(_self_) + .and_then(|c| c.arguments.get(_name).and_then(|v| Some(v.clone()) )) } fn graph_context_add_input_tensor( @@ -660,7 +772,22 @@ impl runtime_v1::RuntimeV1 for Runtime { _element_type: ElementType, _dimensions: DimensionsParam<'_>, ) { - todo!() + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(_self_) + .and_then(|c| { + Some(c.input_tensors.push( + TensorConstraint { + name: _name.to_string(), + element_type: _element_type, + dimensions: match _dimensions { + DimensionsParam::Dynamic => Dimensions::Dynamic, + DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) + } + })) + }); } fn graph_context_add_output_tensor( @@ -670,14 +797,29 @@ impl runtime_v1::RuntimeV1 for Runtime { _element_type: ElementType, _dimensions: DimensionsParam<'_>, ) { - todo!() + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(_self_) + .and_then(|c| { + Some(c.output_tensors.push( + TensorConstraint { + name: _name.to_string(), + element_type: _element_type, + dimensions: match _dimensions { + DimensionsParam::Dynamic => Dimensions::Dynamic, + DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) + } + })) + }); } fn kernel_context_for_node( &mut self, _node_id: &str, ) -> Option { - Some(self.0.clone()) + Some(self.shared_state.clone()) } fn kernel_context_get_argument( From 13576aaa32e9e1e9bbbe25a25bb954020839cf72 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 17 May 2022 06:45:00 +0530 Subject: [PATCH 47/84] Zune: Implement kernel_context functions --- crates/runtime/src/engine/zune.rs | 124 +++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 27 deletions(-) diff --git a/crates/runtime/src/engine/zune.rs b/crates/runtime/src/engine/zune.rs index 0a470edc17..7f22a59593 100644 --- a/crates/runtime/src/engine/zune.rs +++ b/crates/runtime/src/engine/zune.rs @@ -22,7 +22,7 @@ use wasmer::{ }; use zip; -use self::{proc_block_v1::ProcBlockV1, runtime_v1::*}; +use self::{proc_block_v1::*, runtime_v1::*}; use crate::{ callbacks::Callbacks, engine::{host_functions::HostFunctions, LoadError, WebAssemblyEngine}, @@ -40,7 +40,6 @@ struct Runtime { struct State { tensors: Vec>, graph_contexts: HashMap, - tensor_constraints: Vec> } struct ModelNode { @@ -51,8 +50,7 @@ struct ModelNode { } struct ProcBlockNode { - input_tensors: HashMap, - output_tensors: HashMap, + node_id: String, context: ProcBlockV1, shared_state: Arc> } @@ -124,11 +122,11 @@ impl WebAssemblyEngine for ZuneEngine { .iter() .map(|(name, argument)| (name.clone(), argument.to_string())) .collect(); - (k.clone(), GraphContext{ arguments, input_tensors: Vec::new(), output_tensors: Vec::new() }) + (k.clone(), GraphContext{ arguments, input_tensors: HashMap::new(), output_tensors: HashMap::new() }) }) .collect(); - let shared_state = Arc::new(Mutex::new(State { tensors, graph_contexts, tensor_constraints: Vec::new() })); + let shared_state = Arc::new(Mutex::new(State { tensors, graph_contexts })); let (model_contexts, procblock_contexts) = instantiate_nodes( pipeline, @@ -139,6 +137,8 @@ impl WebAssemblyEngine for ZuneEngine { ) .map_err(LoadError::Other)?; + // TODO: Validate and allocate input/output tensors + Ok(ZuneEngine { inputs, outputs, @@ -334,7 +334,7 @@ impl ProcBlockNode { mut imports: &mut ImportObject, shared_state: &Arc>, input_tensors: &HashMap, - output_tensors: &HashMap, + output_tensors: &HashMap ) -> Result { let module = Module::new(&store, wasm).context("Unable to load the module")?; @@ -345,15 +345,46 @@ impl ProcBlockNode { let result = pb.graph(node_id); + // Assign tensors + shared_state.lock() + .unwrap() + .graph_contexts + .get_mut(node_id) + .and_then(|c| { + c.input_tensors.iter_mut() + .enumerate() + .for_each(|(i, (k, t))| { + input_tensors.get(&key(node_id, Some(i))) + .and_then(|&tensor_index| Some(t.tensor_id = Some(tensor_index))); + }); + + c.output_tensors.iter_mut() + .enumerate() + .for_each(|(i, (k, t))| { + output_tensors.get(&key(node_id, Some(i))) + .and_then(|&tensor_index| Some(t.tensor_id = Some(tensor_index))); + }); + Some(()) + }); + Ok(ProcBlockNode { - input_tensors: HashMap::new(), - output_tensors: HashMap::new(), + node_id: node_id.to_string(), context: pb, shared_state: shared_state.clone(), }) } - fn run(&mut self) -> Result<(), Error> { Ok(()) } + fn run(&mut self) -> Result<(), Error> { + self.context + .kernel(&self.node_id) + .map_err(|e| anyhow!("Encountered a Runtime Error"))? + .map_err(|e| match e { + KernelError::Other(s) => anyhow!("Unknown Error"), + KernelError::InvalidArgument(a) => anyhow!("Invalid argument for {}: {}", &self.node_id, a.name), + KernelError::InvalidInput(i) => anyhow!("Invalid input for {}: {}", &self.node_id, i.name), + KernelError::MissingContext => anyhow!("Unable to retrieve kernel context for {}:", &self.node_id) + }) + } } fn get_buffer_size(element_type: ElementType, dimensions: &Vec) -> usize { @@ -584,7 +615,7 @@ enum Dimensions { #[derive(Debug, Clone)] struct TensorConstraint { - name: String, + tensor_id: Option, element_type: ElementType, dimensions: Dimensions } @@ -592,14 +623,14 @@ struct TensorConstraint { #[derive(Debug, Default, Clone)] struct GraphContext { arguments: HashMap, - input_tensors: Vec, - output_tensors: Vec + input_tensors: HashMap, + output_tensors: HashMap } impl runtime_v1::RuntimeV1 for Runtime { type ArgumentHint = Never; type ArgumentMetadata = Never; - type KernelContext = Arc>; + type KernelContext = String; type Metadata = Metadata; type Model = Never; type TensorHint = Never; @@ -778,15 +809,16 @@ impl runtime_v1::RuntimeV1 for Runtime { .graph_contexts .get_mut(_self_) .and_then(|c| { - Some(c.input_tensors.push( + c.input_tensors.insert( + _name.to_string(), TensorConstraint { - name: _name.to_string(), + tensor_id: None, element_type: _element_type, dimensions: match _dimensions { DimensionsParam::Dynamic => Dimensions::Dynamic, DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) } - })) + }) }); } @@ -803,15 +835,16 @@ impl runtime_v1::RuntimeV1 for Runtime { .graph_contexts .get_mut(_self_) .and_then(|c| { - Some(c.output_tensors.push( + c.output_tensors.insert( + _name.to_string(), TensorConstraint { - name: _name.to_string(), + tensor_id: None, element_type: _element_type, dimensions: match _dimensions { DimensionsParam::Dynamic => Dimensions::Dynamic, DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) } - })) + }) }); } @@ -819,28 +852,50 @@ impl runtime_v1::RuntimeV1 for Runtime { &mut self, _node_id: &str, ) -> Option { - Some(self.shared_state.clone()) + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(_node_id) + .and_then(|_| Some(_node_id.to_string())) } fn kernel_context_get_argument( &mut self, - state: &Arc>, + _self_: &Self::KernelContext, name: &str, ) -> Option { - todo!() + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(_self_) + .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()) )) } fn kernel_context_get_input_tensor( &mut self, - state: &Arc>, + _self_: &Self::KernelContext, name: &str, ) -> Option { - todo!() + let state = self.shared_state.lock().unwrap(); + + let tensor_id = state.graph_contexts.get(_self_) + .and_then(|c| { + c.input_tensors + .get(name) + .and_then(|v| v.tensor_id ) + }); + + match tensor_id { + Some(i) => state.tensors[i].clone(), + _ => None + } } fn kernel_context_set_output_tensor( &mut self, - state: &Arc>, + _self_: &Self::KernelContext, name: &str, TensorParam { element_type, @@ -848,7 +903,22 @@ impl runtime_v1::RuntimeV1 for Runtime { dimensions, }: TensorParam<'_>, ) { - todo!() + let mut state = self.shared_state.lock().unwrap(); + + let tensor_id = state.graph_contexts.get(_self_) + .and_then(|c| { + c.output_tensors + .get(name) + .and_then(|v| v.tensor_id ) + }); + + let dimensions = dimensions.iter().map(|&i| i.get() as u32).collect(); + + // Todo check tensor constraint + + if tensor_id.is_some() { + state.tensors[tensor_id.unwrap()] = Some(TensorResult{ element_type, buffer: buffer.to_vec(), dimensions } ); + } } fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { true } From 98e23c246df527b0313b38199cdd3d54d2cd501e Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 15:16:55 +0530 Subject: [PATCH 48/84] Make zune an example for now --- crates/rune-cli/src/run.rs | 12 +- crates/runtime/Cargo.toml | 2 +- crates/runtime/examples/zune_example.rs | 22 ++++ crates/runtime/src/engine/mod.rs | 5 +- crates/runtime/src/lib.rs | 4 +- crates/runtime/src/runtime.rs | 5 - crates/runtime/src/{engine => }/zune.rs | 154 +++++++++++++++++++----- 7 files changed, 155 insertions(+), 49 deletions(-) create mode 100644 crates/runtime/examples/zune_example.rs rename crates/runtime/src/{engine => }/zune.rs (85%) diff --git a/crates/rune-cli/src/run.rs b/crates/rune-cli/src/run.rs index 23864c5d1e..645162104b 100644 --- a/crates/rune-cli/src/run.rs +++ b/crates/rune-cli/src/run.rs @@ -77,9 +77,9 @@ impl Run { format!("Unable to read \"{}\"", self.rune.display()) })?; - if self.rune.extension().unwrap_or(OsStr::new("")).to_ascii_lowercase() == "zune" { - self.engine = Engine::Zune; - } + // if self.rune.extension().unwrap_or(OsStr::new("")).to_ascii_lowercase() == "zune" { + // self.engine = Engine::Zune; + // } let mut runtime: Runtime = self .load_runtime(&rune) @@ -176,8 +176,7 @@ impl Run { ) -> Result { match self.engine { Engine::Wasm3 => Runtime::wasm3(rune), - Engine::Wasmer => Runtime::wasmer(rune), - Engine::Zune => Runtime::zune(rune), + Engine::Wasmer => Runtime::wasmer(rune) } } @@ -257,6 +256,5 @@ fn parse_key_value_pair(s: &str) -> Result<(&str, &str), Error> { #[strum(serialize_all = "kebab-case")] enum Engine { Wasm3, - Wasmer, - Zune + Wasmer } diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 5e17fd430a..b7f85778f0 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -73,7 +73,7 @@ tempfile = "3.2.0" [features] builtins = ["hound", "image", "rand", "rand/small_rng", "csv"] -default = ["builtins", "tflite"] +default = ["builtins", "tflite", "zune"] tflite = ["hotg-runecoral"] unstable_doc_cfg = [] zune = ["wasmer", "zip"] diff --git a/crates/runtime/examples/zune_example.rs b/crates/runtime/examples/zune_example.rs new file mode 100644 index 0000000000..af3bfd9788 --- /dev/null +++ b/crates/runtime/examples/zune_example.rs @@ -0,0 +1,22 @@ +use hotg_rune_runtime::zune::ZuneEngine; + +fn main() { + println!("Hello World!"); + + let sine_zune = include_bytes!("sine.rune"); + let mut zune_engine = ZuneEngine::load(sine_zune).expect("Unable to initialize Zune Engine!"); + + println!("input nodes {:?}", zune_engine.input_nodes()); + println!("output nodes {:?}", zune_engine.output_nodes()); + println!("input tensor names of rand => {:?}", zune_engine.get_input_tensor_names("rand")); + + // let input_tensor = TensorResult { + // element_type: ElementType::f32, + // shape: [1,1], + // buffer: vec![ 0 as u8, 0, 0, 0] + // }; + + println!("input tensor rand => {:?}", zune_engine.get_input_tensor("rand", "input")); + + zune_engine.predict().expect("Failed to run predict!"); +} \ No newline at end of file diff --git a/crates/runtime/src/engine/mod.rs b/crates/runtime/src/engine/mod.rs index 8ca19d82d9..798f0a9465 100644 --- a/crates/runtime/src/engine/mod.rs +++ b/crates/runtime/src/engine/mod.rs @@ -3,8 +3,6 @@ mod host_functions; mod wasm3; #[cfg(feature = "wasmer")] mod wasmer; -#[cfg(feature = "zune")] -mod zune; use std::sync::Arc; @@ -14,8 +12,7 @@ use anyhow::Error; pub(crate) use self::wasm3::Wasm3Engine; #[cfg(feature = "wasmer")] pub(crate) use self::wasmer::WasmerEngine; -#[cfg(feature = "zune")] -pub(crate) use self::zune::ZuneEngine; + /// A WebAssembly virtual machine that links Rune with pub(crate) trait WebAssemblyEngine { fn load( diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 387a5d19e3..3fd6515f57 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -32,10 +32,12 @@ mod tensor; pub mod builtins; mod outputs; +pub mod zune; + pub use crate::{ callbacks::{Model, ModelMetadata, NodeMetadata}, engine::LoadError, outputs::OutputTensor, runtime::Runtime, - tensor::{ElementType, Tensor, TensorElement}, + tensor::{ElementType, Tensor, TensorElement} }; diff --git a/crates/runtime/src/runtime.rs b/crates/runtime/src/runtime.rs index b565649b61..923ceb8478 100644 --- a/crates/runtime/src/runtime.rs +++ b/crates/runtime/src/runtime.rs @@ -70,11 +70,6 @@ impl Runtime { Runtime::load::(rune) } - #[cfg(feature = "zune")] - pub fn zune(rune: &[u8]) -> Result { - Runtime::load::(rune) - } - fn load(rune: &[u8]) -> Result where E: WebAssemblyEngine + 'static, diff --git a/crates/runtime/src/engine/zune.rs b/crates/runtime/src/zune.rs similarity index 85% rename from crates/runtime/src/engine/zune.rs rename to crates/runtime/src/zune.rs index 7f22a59593..98f113ae47 100644 --- a/crates/runtime/src/engine/zune.rs +++ b/crates/runtime/src/zune.rs @@ -24,8 +24,7 @@ use zip; use self::{proc_block_v1::*, runtime_v1::*}; use crate::{ - callbacks::Callbacks, - engine::{host_functions::HostFunctions, LoadError, WebAssemblyEngine}, + LoadError, }; wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); @@ -39,6 +38,7 @@ struct Runtime { #[derive(Debug, Default)] struct State { tensors: Vec>, + tensor_constraints: Vec>, graph_contexts: HashMap, } @@ -56,8 +56,8 @@ struct ProcBlockNode { } pub struct ZuneEngine { - inputs: Vec, - outputs: Vec, + input_nodes: Vec, + output_nodes: Vec, models: HashMap, procblocks: HashMap, pipeline: IndexMap, @@ -65,8 +65,8 @@ pub struct ZuneEngine { shared_state: Arc>, // resources } -impl WebAssemblyEngine for ZuneEngine { - fn load(binary: &[u8], _: Arc) -> Result +impl ZuneEngine { + pub fn load(binary: &[u8]) -> Result where Self: Sized, { @@ -126,7 +126,13 @@ impl WebAssemblyEngine for ZuneEngine { }) .collect(); - let shared_state = Arc::new(Mutex::new(State { tensors, graph_contexts })); + let tensor_constraints = tensors.iter().map(|_| None).collect(); + let shared_state = Arc::new(Mutex::new(State { tensors, tensor_constraints, graph_contexts })); + + println!("input_tensors {:?} ", &input_tensors); + println!("output_tensors {:?} ", &output_tensors); + // println!("tensors: {:?}", &tensors); + let (model_contexts, procblock_contexts) = instantiate_nodes( pipeline, @@ -140,8 +146,8 @@ impl WebAssemblyEngine for ZuneEngine { // TODO: Validate and allocate input/output tensors Ok(ZuneEngine { - inputs, - outputs, + input_nodes: inputs, + output_nodes: outputs, models: model_contexts, procblocks: procblock_contexts, pipeline: pipeline.to_owned(), @@ -150,21 +156,14 @@ impl WebAssemblyEngine for ZuneEngine { }) } - fn init(&mut self) -> Result<(), Error> { - // TODO: Call each proc block's graph() and each model's inputs/outputs - // to allocate the tensors with correct dimensions - - Ok(()) - } - - fn predict(&mut self) -> Result<(), Error> { + pub fn predict(&mut self) -> Result<(), Error> { for stage_name in &self.processing_order { let stage = self.pipeline.get(stage_name).unwrap(); match stage { - Stage::Model(stage) => { + Stage::Model(_) => { self.models.get_mut(stage_name).unwrap().run()?; }, - Stage::ProcBlock(stage) => { + Stage::Capability(_) | Stage::ProcBlock(_) => { self.procblocks.get_mut(stage_name).unwrap().run()?; }, _ => {}, @@ -172,6 +171,80 @@ impl WebAssemblyEngine for ZuneEngine { } Ok(()) } + + pub fn input_nodes(&self) -> &'_ Vec { + return &self.input_nodes; + } + + pub fn output_nodes(&self) -> &'_ Vec { + return &self.output_nodes; + } + + pub fn get_input_tensor_names(&self, node_name: &str) -> Result, Error> { + let state = self.shared_state.lock().unwrap(); + state.graph_contexts + .get(node_name) + .and_then(|c| { + let tensor_list: Vec = c.input_tensors.iter() + .map(|(k, _)| k.to_string()) + .collect(); + Some(tensor_list) + }) + .ok_or(anyhow!("Unable to get input tensors")) + } + + pub fn get_input_tensor(&mut self, node_name: &str, tensor_name: &str) -> Option { + let state = self.shared_state.lock().unwrap(); + let tensor_constraint = state.graph_contexts.get(node_name).and_then(|c| c.input_tensors.get(tensor_name)); + + match tensor_constraint { + Some(c) if c.tensor_id.is_some() => state.tensors[c.tensor_id.unwrap()].clone(), + _ => None + } + } + + pub fn set_input_tensor(&mut self, node_name: &str, tensor_name: &str, tensor: &TensorResult) { + let mut state = self.shared_state.lock().unwrap(); + let tensor_id = state.graph_contexts.get(node_name).and_then(|c| c.input_tensors.get(tensor_name).and_then(|c| c.tensor_id.clone())); + + match tensor_id { + Some(i) => state.tensors[i] = Some(tensor.clone()), + _ => {} + } + } + + pub fn get_output_tensor_names(&self, node_name: &str) -> Result, Error> { + let state = self.shared_state.lock().unwrap(); + state.graph_contexts + .get(node_name) + .and_then(|c| { + let tensor_list: Vec = c.output_tensors.iter() + .map(|(k, _)| k.to_string()) + .collect(); + Some(tensor_list) + }) + .ok_or(anyhow!("Unable to get input tensors")) + } + + pub fn get_output_tensor(&mut self, node_name: &str, tensor_name: &str) -> Option { + let state = self.shared_state.lock().unwrap(); + let tensor_constraint = state.graph_contexts.get(node_name).and_then(|c| c.output_tensors.get(tensor_name)); + + match tensor_constraint { + Some(c) if c.tensor_id.is_some() => state.tensors[c.tensor_id.unwrap()].clone(), + _ => None + } + } + + pub fn set_output_tensor(&mut self, node_name: &str, tensor_name: &str, tensor: &TensorResult) { + let mut state = self.shared_state.lock().unwrap(); + let tensor_id = state.graph_contexts.get(node_name).and_then(|c| c.output_tensors.get(tensor_name).and_then(|c| c.tensor_id.clone())); + + match tensor_id { + Some(i) => state.tensors[i] = Some(tensor.clone()), + _ => {} + } + } } impl ModelNode { @@ -328,7 +401,6 @@ impl ModelNode { impl ProcBlockNode { fn load( node_id: &str, - node_data: &ProcBlockStage, wasm: &[u8], store: &Store, mut imports: &mut ImportObject, @@ -346,6 +418,8 @@ impl ProcBlockNode { let result = pb.graph(node_id); // Assign tensors + // TODO: See if this can be more smart. + // Not bothering with that for now because tensor names are lost in current Runefile format shared_state.lock() .unwrap() .graph_contexts @@ -451,8 +525,23 @@ fn instantiate_nodes( // Collect each output tensor into tensors let stage_name = item.0; match item.1 { - Stage::Capability(_) => { - // inputs.push(stage_name.to_string()); + // Models are handled on the host side, so we treat them separately + Stage::Capability(stage) => { + let wasm = read_zip_resource_by_path(&stage.capability) + .context("Unable to load the capability")?; + + procblocks.insert( + stage_name.to_string(), + ProcBlockNode::load( + &stage_name, + &wasm, + &store, + &mut imports, + &shared_state, + &input_tensors, + &output_tensors, + )?, + ); }, Stage::Model(stage) => { // Instantiating the model's inference context here because that @@ -480,10 +569,6 @@ fn instantiate_nodes( ); }, Stage::ProcBlock(stage) => { - println!( - "Pipeline stage: {} proc block {:?} {}", - stage_name, stage.args, stage.proc_block.base - ); let wasm = read_zip_resource_by_path(&stage.proc_block.base) .context("Unable to load the proc_block")?; @@ -491,7 +576,6 @@ fn instantiate_nodes( stage_name.to_string(), ProcBlockNode::load( &stage_name, - &stage, &wasm, &store, &mut imports, @@ -501,9 +585,8 @@ fn instantiate_nodes( )?, ); }, - Stage::Out(_) => { - // outputs.push(stage_name.to_string()); - }, + + _ => { } // Do nothing for capabilities/outputs } } @@ -529,13 +612,22 @@ fn get_tensors( let mut output_tensors: HashMap = HashMap::new(); let mut input_tensors: HashMap = HashMap::new(); - // For Inputs/Capabilities - input tensors and output tensors are the same? + // For Inputs/Capabilities - We create an input so as to be able to inject inputs for item in inputs { tensors.push(None); input_tensors.insert(key(item, Some(0)), tensors.len() - 1); output_tensors.insert(key(item, Some(0)), tensors.len() - 1); } + // // For Outputs - we allocate all the outputs + // for item in outputs { + // for _ in pipeline.get(item).unwrap().output_types() { + // tensors.push(None); + // output_tensors.insert(key(item, Some(0)), tensors.len() - 1); + // } + // } + + // Do a depth first traversal of the tree structure to determine the order // of processing/calling predict() Also allocate the output tensors of // each node along the way From d91fb97ac633e93d29520741eea20b09329cc5cf Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Fri, 20 May 2022 20:13:31 +0530 Subject: [PATCH 49/84] Zune: Fix bug - Do not share Store, import objects among proc blocks --- crates/runtime/src/zune.rs | 43 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 98f113ae47..357ef8e6f8 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -22,7 +22,7 @@ use wasmer::{ }; use zip; -use self::{proc_block_v1::*, runtime_v1::*}; +pub use self::{proc_block_v1::*, runtime_v1::*}; use crate::{ LoadError, }; @@ -143,6 +143,8 @@ impl ZuneEngine { ) .map_err(LoadError::Other)?; + println!(" execution order: {:?}", processing_order); + // TODO: Validate and allocate input/output tensors Ok(ZuneEngine { @@ -402,15 +404,17 @@ impl ProcBlockNode { fn load( node_id: &str, wasm: &[u8], - store: &Store, - mut imports: &mut ImportObject, - shared_state: &Arc>, + runtime: &Runtime, input_tensors: &HashMap, output_tensors: &HashMap ) -> Result { - let module = - Module::new(&store, wasm).context("Unable to load the module")?; + let shared_state = runtime.shared_state.clone(); + let store = Store::default(); + let mut imports = ImportObject::default(); + add_to_imports(&store, &mut imports, runtime.clone()); + let module = + Module::new(&store, wasm).context("Unable to load the module")?; let (pb, _) = ProcBlockV1::instantiate(&store, &module, &mut imports) .context("Unable to instantiate the WebAssembly module")?; @@ -449,11 +453,13 @@ impl ProcBlockNode { } fn run(&mut self) -> Result<(), Error> { + println!("Executing proc block: {:?} ", self.node_id); + // impl stderr for KernelError self.context .kernel(&self.node_id) .map_err(|e| anyhow!("Encountered a Runtime Error"))? .map_err(|e| match e { - KernelError::Other(s) => anyhow!("Unknown Error"), + KernelError::Other(s) => anyhow!(s), KernelError::InvalidArgument(a) => anyhow!("Invalid argument for {}: {}", &self.node_id, a.name), KernelError::InvalidInput(i) => anyhow!("Invalid input for {}: {}", &self.node_id, i.name), KernelError::MissingContext => anyhow!("Unable to retrieve kernel context for {}:", &self.node_id) @@ -516,10 +522,7 @@ fn instantiate_nodes( let mut models: HashMap = HashMap::new(); let mut procblocks: HashMap = HashMap::new(); - let store = Store::default(); - let mut imports = ImportObject::default(); let mut runtime = Runtime{ shared_state: shared_state.clone() }; - add_to_imports(&store, &mut imports, runtime.clone()); for item in pipeline { // Collect each output tensor into tensors @@ -535,9 +538,7 @@ fn instantiate_nodes( ProcBlockNode::load( &stage_name, &wasm, - &store, - &mut imports, - &shared_state, + &runtime, &input_tensors, &output_tensors, )?, @@ -577,9 +578,7 @@ fn instantiate_nodes( ProcBlockNode::load( &stage_name, &wasm, - &store, - &mut imports, - &shared_state, + &runtime, &input_tensors, &output_tensors, )?, @@ -871,8 +870,9 @@ impl runtime_v1::RuntimeV1 for Runtime { .lock() .unwrap() .graph_contexts - .get(_node_id) - .and_then(|_| Some(_node_id.to_string())) + .get(_node_id)?; + + Some(_node_id.to_string()) } fn graph_context_get_argument( @@ -944,12 +944,11 @@ impl runtime_v1::RuntimeV1 for Runtime { &mut self, _node_id: &str, ) -> Option { - self.shared_state - .lock() + self.shared_state.lock() .unwrap() .graph_contexts - .get(_node_id) - .and_then(|_| Some(_node_id.to_string())) + .get(_node_id)?; + Some(_node_id.to_string()) } fn kernel_context_get_argument( From 08afed086a48781effa27d5f11d02969df9e89f6 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Fri, 20 May 2022 20:13:40 +0530 Subject: [PATCH 50/84] Zune update example --- crates/runtime/examples/zune_example.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/runtime/examples/zune_example.rs b/crates/runtime/examples/zune_example.rs index af3bfd9788..816cb75c32 100644 --- a/crates/runtime/examples/zune_example.rs +++ b/crates/runtime/examples/zune_example.rs @@ -1,4 +1,4 @@ -use hotg_rune_runtime::zune::ZuneEngine; +use hotg_rune_runtime::zune::{ZuneEngine, TensorResult, ElementType}; fn main() { println!("Hello World!"); @@ -9,14 +9,19 @@ fn main() { println!("input nodes {:?}", zune_engine.input_nodes()); println!("output nodes {:?}", zune_engine.output_nodes()); println!("input tensor names of rand => {:?}", zune_engine.get_input_tensor_names("rand")); + println!("output tensor names of sine => {:?}", zune_engine.get_output_tensor_names("sine")); - // let input_tensor = TensorResult { - // element_type: ElementType::f32, - // shape: [1,1], - // buffer: vec![ 0 as u8, 0, 0, 0] - // }; + let input_tensor = TensorResult { + element_type: ElementType::F32, + dimensions: vec![1,1], + buffer: vec![ 0, 0, 0, 0] + }; + + zune_engine.set_input_tensor("rand", "input", &input_tensor); println!("input tensor rand => {:?}", zune_engine.get_input_tensor("rand", "input")); zune_engine.predict().expect("Failed to run predict!"); + + println!("input tensor for serial: => {:?}", zune_engine.get_input_tensor("serial", "input")); } \ No newline at end of file From e16db204fd0a3fd9ed5c525a2b3390b1c04c2327 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Wed, 25 May 2022 15:45:51 +0530 Subject: [PATCH 51/84] zune_example: Reformat --- crates/runtime/examples/zune_example.rs | 39 ++++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/crates/runtime/examples/zune_example.rs b/crates/runtime/examples/zune_example.rs index 816cb75c32..6f478a563e 100644 --- a/crates/runtime/examples/zune_example.rs +++ b/crates/runtime/examples/zune_example.rs @@ -1,27 +1,38 @@ -use hotg_rune_runtime::zune::{ZuneEngine, TensorResult, ElementType}; +use hotg_rune_runtime::zune::{ElementType, TensorResult, ZuneEngine}; fn main() { - println!("Hello World!"); - let sine_zune = include_bytes!("sine.rune"); - let mut zune_engine = ZuneEngine::load(sine_zune).expect("Unable to initialize Zune Engine!"); - - println!("input nodes {:?}", zune_engine.input_nodes()); - println!("output nodes {:?}", zune_engine.output_nodes()); - println!("input tensor names of rand => {:?}", zune_engine.get_input_tensor_names("rand")); - println!("output tensor names of sine => {:?}", zune_engine.get_output_tensor_names("sine")); + let mut zune_engine = + ZuneEngine::load(sine_zune).expect("Unable to initialize Zune Engine!"); + + println!("input nodes {:?}", zune_engine.input_nodes()); + println!("output nodes {:?}", zune_engine.output_nodes()); + println!( + "input tensor names of rand => {:?}", + zune_engine.get_input_tensor_names("rand") + ); + println!( + "output tensor names of sine => {:?}", + zune_engine.get_output_tensor_names("sine") + ); let input_tensor = TensorResult { element_type: ElementType::F32, - dimensions: vec![1,1], - buffer: vec![ 0, 0, 0, 0] + dimensions: vec![1, 1], + buffer: vec![0, 0, 0, 0], }; zune_engine.set_input_tensor("rand", "input", &input_tensor); - println!("input tensor rand => {:?}", zune_engine.get_input_tensor("rand", "input")); + println!( + "input tensor rand => {:?}", + zune_engine.get_input_tensor("rand", "input") + ); zune_engine.predict().expect("Failed to run predict!"); - println!("input tensor for serial: => {:?}", zune_engine.get_input_tensor("serial", "input")); -} \ No newline at end of file + println!( + "input tensor for serial: => {:?}", + zune_engine.get_output_tensor("sine", "0") + ); +} From e2576d666ea7a526e20f8c75d4627418b82df7e8 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Wed, 25 May 2022 15:46:46 +0530 Subject: [PATCH 52/84] zune: Add GraphContext for Model nodes --- crates/runtime/src/zune.rs | 42 ++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 357ef8e6f8..5f8ba8c8cd 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -131,8 +131,6 @@ impl ZuneEngine { println!("input_tensors {:?} ", &input_tensors); println!("output_tensors {:?} ", &output_tensors); - // println!("tensors: {:?}", &tensors); - let (model_contexts, procblock_contexts) = instantiate_nodes( pipeline, @@ -284,19 +282,32 @@ impl ModelNode { } }; + let tensor_constraint_from_descriptor = + |t: &RuneCoralTensorDescriptor, tensor_id: usize| -> TensorConstraint { + let element_type = get_element_type(t); + let dimensions = t.shape.iter().map(|&x| x as u32).collect(); + let buffer_size = get_buffer_size(element_type, &dimensions); + + TensorConstraint { + tensor_id: Some(tensor_id), + element_type, + dimensions: Dimensions::Fixed(dimensions.iter().map(|&x| x as usize).collect()), + } + }; + // Returns the list of tensor indices in the State's tensors let allocate_tensors = |tensor_type: &str, model_tensors: &mut dyn Iterator< Item = RuneCoralTensorDescriptor, >, pipeline_tensors: &HashMap| - -> Result, Error> { - let mut result: HashSet = HashSet::new(); + -> Result<(HashSet, HashMap), Error> { + let mut tensor_indices: HashSet = HashSet::new(); + let mut tensor_constraints: HashMap = HashMap::new(); let mut i = 0; let mut s = shared_state.lock().unwrap(); while let Some(model_tensor) = model_tensors.next() { - let model_tensor = tensor_from_descriptor(&model_tensor); let tensor_key = key(&node_id, Some(i)); let tensor_id = *pipeline_tensors.get(&tensor_key).ok_or_else(|| { @@ -308,6 +319,9 @@ impl ModelNode { ) })?; + let tensor_constraint = tensor_constraint_from_descriptor(&model_tensor, tensor_id); + let model_tensor = tensor_from_descriptor(&model_tensor); + match s.tensors[tensor_id] { Some(ref t) if t.dimensions != model_tensor.dimensions @@ -326,22 +340,32 @@ impl ModelNode { }, } - result.insert(tensor_id); + tensor_indices.insert(tensor_id); + tensor_constraints.insert(format!("{}", i), tensor_constraint); i += 1; } - Ok(result) + Ok((tensor_indices, tensor_constraints)) }; - let input_tensors = + let (input_tensors, input_tensor_constraints) = allocate_tensors("input", &mut context.inputs(), &input_tensors)?; - let output_tensors = allocate_tensors( + + let (output_tensors, output_tensor_constraints) = allocate_tensors( "output", &mut context.outputs(), &output_tensors, )?; + let graph_context = GraphContext { + arguments: node_data.args.iter().map(|(k, v)| (k.clone(), v.to_string())).collect(), + input_tensors: input_tensor_constraints, + output_tensors: output_tensor_constraints + }; + + shared_state.lock().unwrap().graph_contexts.insert(node_id.to_string(), graph_context); + Ok(ModelNode { context, input_tensors, From e0018243de04d1b4327a6766d85875b70e97eb7f Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Fri, 27 May 2022 20:43:28 +0530 Subject: [PATCH 53/84] Github Actions: Checkout submodules --- .github/workflows/main.yml | 6 ++++++ .github/workflows/releases.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 19396a4aaa..b9a61d8f1a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,6 +20,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 + with: + submodules: true - uses: actions/cache@v2 with: path: | @@ -77,6 +79,8 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + with: + submodules: true - uses: actions/cache@v2 with: path: | @@ -137,6 +141,8 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 + with: + submodules: true - uses: actions/cache@v2 with: path: | diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index bbb763cd82..fe3e039150 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -20,6 +20,7 @@ jobs: steps: - uses: actions/checkout@v2 with: + submodules: true fetch-depth: 0 - name: print latest_commit run: echo ${{ github.sha }} @@ -61,6 +62,8 @@ jobs: asset_name: rune-macos steps: - uses: actions/checkout@v1 + with: + submodules: true - uses: actions/cache@v2 with: path: | @@ -163,6 +166,8 @@ jobs: echo "tag=nightly" >> $GITHUB_ENV - name: Checkout code uses: actions/checkout@v2 + with: + submodules: true - name: Fetch Pre-Compiled Binaries uses: actions/download-artifact@v2 with: From 23a7262076283beacf3e0fbfb69830d88df934a0 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Fri, 27 May 2022 20:44:47 +0530 Subject: [PATCH 54/84] Cleanup zune.rs and zune_example.rs --- crates/runtime/examples/zune_example.rs | 8 ++- crates/runtime/src/zune.rs | 65 ++++++++++++++++--------- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/crates/runtime/examples/zune_example.rs b/crates/runtime/examples/zune_example.rs index 6f478a563e..ec1fdf6e90 100644 --- a/crates/runtime/examples/zune_example.rs +++ b/crates/runtime/examples/zune_example.rs @@ -11,6 +11,10 @@ fn main() { "input tensor names of rand => {:?}", zune_engine.get_input_tensor_names("rand") ); + println!( + "input tensor names of sine => {:?}", + zune_engine.get_input_tensor_names("sine") + ); println!( "output tensor names of sine => {:?}", zune_engine.get_output_tensor_names("sine") @@ -32,7 +36,7 @@ fn main() { zune_engine.predict().expect("Failed to run predict!"); println!( - "input tensor for serial: => {:?}", - zune_engine.get_output_tensor("sine", "0") + "output tensor for sine: => {:?}", + zune_engine.get_output_tensor("sine", "Identity") ); } diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 5f8ba8c8cd..bbe8607066 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -1,25 +1,20 @@ use std::{ borrow::Cow, collections::{HashMap, HashSet}, - convert::TryInto, - fmt::{self, Display, Formatter}, io::{Cursor, Read}, sync::{Arc, Mutex}, }; use anyhow::{anyhow, Context, Error}; -use hotg_rune_compiler::{diagnostics::Diagnostics, parse::yaml::*}; -use hotg_rune_core::{ElementType as RuneElementType, Shape, TFLITE_MIMETYPE}; +use hotg_rune_compiler::parse::yaml::*; +use hotg_rune_core::{TFLITE_MIMETYPE}; use hotg_runecoral::{ AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, Tensor as RuneCoralTensor, TensorDescriptor as RuneCoralTensorDescriptor, TensorMut as RuneCoralTensorMut, }; use indexmap::IndexMap; -use wasmer::{ - Array, Function, ImportObject, Instance, LazyInit, Memory, Module, - NativeFunc, RuntimeError, Store, ValueType, WasmPtr, WasmerEnv, -}; +use wasmer::{ImportObject, Module, Store}; use zip; pub use self::{proc_block_v1::*, runtime_v1::*}; @@ -236,6 +231,26 @@ impl ZuneEngine { } } + // pub fn get_tensor(&self, tensor_id: usize) -> Option<&TensorResult> { + // self.shared_state + // .lock() + // .unwrap() + // .tensors + // .get(tensor_id) + // .unwrap_or(&None) + // .as_ref() + // } + + // pub fn set_tensor(&mut self, tensor_id: usize, tensor: &TensorResult) -> Result<(), Error> { + // self.shared_state + // .lock() + // .unwrap() + // .tensors + // .get_mut(tensor_id) + // .and_then(|t| { t = Some(tensor.clone()); Ok() }) + // .ok() + // } + pub fn set_output_tensor(&mut self, node_name: &str, tensor_name: &str, tensor: &TensorResult) { let mut state = self.shared_state.lock().unwrap(); let tensor_id = state.graph_contexts.get(node_name).and_then(|c| c.output_tensors.get(tensor_name).and_then(|c| c.tensor_id.clone())); @@ -285,13 +300,12 @@ impl ModelNode { let tensor_constraint_from_descriptor = |t: &RuneCoralTensorDescriptor, tensor_id: usize| -> TensorConstraint { let element_type = get_element_type(t); - let dimensions = t.shape.iter().map(|&x| x as u32).collect(); - let buffer_size = get_buffer_size(element_type, &dimensions); + let dimensions = t.shape.iter().map(|&x| x as usize).collect(); TensorConstraint { tensor_id: Some(tensor_id), element_type, - dimensions: Dimensions::Fixed(dimensions.iter().map(|&x| x as usize).collect()), + dimensions: Dimensions::Fixed(dimensions), } }; @@ -319,6 +333,11 @@ impl ModelNode { ) })?; + let tensor_name = model_tensor.name.to_str().ok(); + let tensor_name = match tensor_name { + Some(tensor_name) if tensor_name.len() > 0 => tensor_name.to_string(), + _ => format!("{}", i).to_string() + }; let tensor_constraint = tensor_constraint_from_descriptor(&model_tensor, tensor_id); let model_tensor = tensor_from_descriptor(&model_tensor); @@ -341,7 +360,7 @@ impl ModelNode { } tensor_indices.insert(tensor_id); - tensor_constraints.insert(format!("{}", i), tensor_constraint); + tensor_constraints.insert(tensor_name , tensor_constraint); i += 1; } @@ -385,7 +404,7 @@ impl ModelNode { state.tensors.iter_mut().enumerate().for_each(|(i, t)| { if self.input_tensors.contains(&i) { - let mut pipeline_tensor = t.as_mut().unwrap(); + let pipeline_tensor = t.as_mut().unwrap(); unsafe { inputs.push(RuneCoralTensor { element_type: get_runecoral_element_type( @@ -399,7 +418,7 @@ impl ModelNode { }) } } else if self.output_tensors.contains(&i) { - let mut pipeline_tensor = t.as_mut().unwrap(); + let pipeline_tensor = t.as_mut().unwrap(); unsafe { outputs.push(RuneCoralTensorMut { element_type: get_runecoral_element_type( @@ -443,7 +462,7 @@ impl ProcBlockNode { ProcBlockV1::instantiate(&store, &module, &mut imports) .context("Unable to instantiate the WebAssembly module")?; - let result = pb.graph(node_id); + let _result = pb.graph(node_id); // Assign tensors // TODO: See if this can be more smart. @@ -455,14 +474,14 @@ impl ProcBlockNode { .and_then(|c| { c.input_tensors.iter_mut() .enumerate() - .for_each(|(i, (k, t))| { + .for_each(|(i, (_, t))| { input_tensors.get(&key(node_id, Some(i))) .and_then(|&tensor_index| Some(t.tensor_id = Some(tensor_index))); }); c.output_tensors.iter_mut() .enumerate() - .for_each(|(i, (k, t))| { + .for_each(|(i, (_, t))| { output_tensors.get(&key(node_id, Some(i))) .and_then(|&tensor_index| Some(t.tensor_id = Some(tensor_index))); }); @@ -481,7 +500,7 @@ impl ProcBlockNode { // impl stderr for KernelError self.context .kernel(&self.node_id) - .map_err(|e| anyhow!("Encountered a Runtime Error"))? + .map_err(|_| anyhow!("Encountered a Runtime Error"))? .map_err(|e| match e { KernelError::Other(s) => anyhow!(s), KernelError::InvalidArgument(a) => anyhow!("Invalid argument for {}: {}", &self.node_id, a.name), @@ -546,7 +565,7 @@ fn instantiate_nodes( let mut models: HashMap = HashMap::new(); let mut procblocks: HashMap = HashMap::new(); - let mut runtime = Runtime{ shared_state: shared_state.clone() }; + let runtime = Runtime{ shared_state: shared_state.clone() }; for item in pipeline { // Collect each output tensor into tensors @@ -1076,7 +1095,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn kernel_context_get_global_input( &mut self, - self_: &Self::KernelContext, + _self_: &Self::KernelContext, name: &str, ) -> Option { todo!() @@ -1084,9 +1103,9 @@ impl runtime_v1::RuntimeV1 for Runtime { fn kernel_context_set_global_output( &mut self, - self_: &Self::KernelContext, - name: &str, - tensor: TensorParam<'_>, + _self_: &Self::KernelContext, + _name: &str, + _tensor: TensorParam<'_>, ) { todo!() } From a59497ceaf5fe46ce8df3a3cb0906e2cfe16da74 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 14:35:45 +0530 Subject: [PATCH 55/84] Zune: get_tensors: Do not add the same node twice to visit --- crates/runtime/src/zune.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index bbe8607066..895419331c 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -360,6 +360,8 @@ impl ModelNode { } tensor_indices.insert(tensor_id); + //FIXME: 2 tensors share same name (/empty name) + //then tensor_indices.len() != tensor_constraints.len() tensor_constraints.insert(tensor_name , tensor_constraint); i += 1; @@ -685,7 +687,7 @@ fn get_tensors( } for input in stage.inputs() { - if !nodes_visited.contains(&input.name) { + if !nodes_to_visit.contains(input) && !nodes_visited.contains(&input.name) { nodes_to_visit.push(input.name.clone()); } } From 4f29f8d4729f700f1eeb5c797ebdcc7c24d241c5 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 14:36:05 +0530 Subject: [PATCH 56/84] Update librunecoral --- crates/rune-cli/Cargo.toml | 2 +- crates/runtime/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rune-cli/Cargo.toml b/crates/rune-cli/Cargo.toml index 0ca73c24a4..de1d3c6744 100644 --- a/crates/rune-cli/Cargo.toml +++ b/crates/rune-cli/Cargo.toml @@ -27,7 +27,7 @@ hotg-rune-compiler = "^0.11.0" hotg-rune-core = "^0.11.0" hotg-rune-proc-blocks = { version = "0.11.3", path = "../proc-blocks" } hotg-rune-runtime = { path = "../runtime", version = "^0.11.0", features = ["builtins", "wasm3", "wasmer", "zune"] } -hotg-runecoral = "0.3.11" +hotg-runecoral = "0.3.12" hound = "3.4.0" human-panic = "1.0.3" image = "0.23.14" diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index b7f85778f0..ac1e8f2bcb 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -35,7 +35,7 @@ version = "^0.11.0" [dependencies.hotg-runecoral] optional = true -version = "0.3.11" +version = "0.3.12" [dependencies.hound] optional = true From e04f3dff8a72f70edf37d5bb4b038e8865644b7d Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 14:41:08 +0530 Subject: [PATCH 57/84] Zune: Fix for the previous commit --- crates/runtime/src/zune.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 895419331c..6cd75faa8d 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -687,7 +687,7 @@ fn get_tensors( } for input in stage.inputs() { - if !nodes_to_visit.contains(input) && !nodes_visited.contains(&input.name) { + if !nodes_to_visit.contains(&input.name) && !nodes_visited.contains(&input.name) { nodes_to_visit.push(input.name.clone()); } } From 9b16f6062ec6e3cb540c5410b163e056030b9f33 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Tue, 31 May 2022 15:59:24 +0530 Subject: [PATCH 58/84] Zune: Fix Compiler breakages --- crates/runtime/src/zune.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 6cd75faa8d..4a49dccb14 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -575,7 +575,7 @@ fn instantiate_nodes( match item.1 { // Models are handled on the host side, so we treat them separately Stage::Capability(stage) => { - let wasm = read_zip_resource_by_path(&stage.capability) + let wasm = read_zip_resource_by_path(&stage.capability.to_string()) .context("Unable to load the capability")?; procblocks.insert( @@ -615,7 +615,7 @@ fn instantiate_nodes( ); }, Stage::ProcBlock(stage) => { - let wasm = read_zip_resource_by_path(&stage.proc_block.base) + let wasm = read_zip_resource_by_path(&stage.proc_block.to_string()) .context("Unable to load the proc_block")?; procblocks.insert( From b18ec5e98a678924f9ce57da0877f6b9523918b7 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 31 May 2022 19:56:47 +0800 Subject: [PATCH 59/84] Made the zune example compile --- crates/runtime/examples/zune_example.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/runtime/examples/zune_example.rs b/crates/runtime/examples/zune_example.rs index ec1fdf6e90..4e45230363 100644 --- a/crates/runtime/examples/zune_example.rs +++ b/crates/runtime/examples/zune_example.rs @@ -1,9 +1,16 @@ +use anyhow::{Context, Error}; use hotg_rune_runtime::zune::{ElementType, TensorResult, ZuneEngine}; -fn main() { - let sine_zune = include_bytes!("sine.rune"); - let mut zune_engine = - ZuneEngine::load(sine_zune).expect("Unable to initialize Zune Engine!"); +fn main() -> Result<(), Error> { + let args: Vec = std::env::args().collect(); + + let filename = args.get(1).map(|s| s.as_str()).unwrap_or("sine.rune"); + + let sine_zune = std::fs::read(&filename) + .with_context(|| format!("Unable to read \"{filename}\""))?; + + let mut zune_engine = ZuneEngine::load(&sine_zune) + .context("Unable to initialize Zune Engine!")?; println!("input nodes {:?}", zune_engine.input_nodes()); println!("output nodes {:?}", zune_engine.output_nodes()); @@ -33,10 +40,12 @@ fn main() { zune_engine.get_input_tensor("rand", "input") ); - zune_engine.predict().expect("Failed to run predict!"); + zune_engine.predict().context("Failed to run predict!")?; println!( "output tensor for sine: => {:?}", zune_engine.get_output_tensor("sine", "Identity") ); + + Ok(()) } From e07927676b68b8d121b3ceabf5d8a7d09545f250 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Jun 2022 03:18:00 +0800 Subject: [PATCH 60/84] Copying the basic FileSystem implementation into hotg-rune-compiler --- Cargo.lock | 1 + crates/compiler/Cargo.toml | 6 + crates/compiler/examples/compile.rs | 36 +-- crates/compiler/src/codegen/query.rs | 6 +- crates/compiler/src/filesystem.rs | 53 ---- crates/compiler/src/filesystem/builtin.rs | 107 ++++++++ crates/compiler/src/filesystem/mod.rs | 288 ++++++++++++++++++++++ crates/compiler/src/lib.rs | 3 +- crates/compiler/src/parse/query.rs | 6 +- 9 files changed, 416 insertions(+), 90 deletions(-) delete mode 100644 crates/compiler/src/filesystem.rs create mode 100644 crates/compiler/src/filesystem/builtin.rs create mode 100644 crates/compiler/src/filesystem/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 3b8d13acac..1e920a7bef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1371,6 +1371,7 @@ dependencies = [ "jsonschema", "once_cell", "pretty_assertions 1.2.1", + "queryst", "regex 1.5.6", "reqwest", "salsa", diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index b10f8b0fff..dd5eef85a3 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -11,7 +11,9 @@ readme = "README.md" [dependencies] indexmap = { version = "1.8.1", features = ["serde-1"] } once_cell = "1.10.0" +queryst = { version = "2.1.0" } regex = "1.5.5" +reqwest = { version = "0.11.10", features = ["blocking"], optional = true } salsa = "0.16.1" schemars = { version = "0.8.8", features = ["indexmap"] } serde = "1.0.136" @@ -29,3 +31,7 @@ pretty_assertions = "1.2.1" reqwest = "0.11.10" tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } tracing-test = "0.2.1" + +[features] +default = ["builtins"] +builtins = ["reqwest"] diff --git a/crates/compiler/examples/compile.rs b/crates/compiler/examples/compile.rs index 2979126d99..b9298a7300 100644 --- a/crates/compiler/examples/compile.rs +++ b/crates/compiler/examples/compile.rs @@ -2,13 +2,13 @@ use std::path::PathBuf; use hotg_rune_compiler::{ codegen::{Codegen, CodegenStorage}, + filesystem::{FileSystem, ReadError, StandardFileSystem}, im::Vector, parse::{Frontend, FrontendStorage}, - BuildConfig, Environment, EnvironmentStorage, FeatureFlags, FileSystem, - ReadError, + BuildConfig, Environment, EnvironmentStorage, FeatureFlags, }; use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; -use uriparse::{Scheme, URI}; +use uriparse::URI; fn main() { tracing_subscriber::fmt() @@ -45,6 +45,7 @@ fn main() { #[salsa::database(FrontendStorage, EnvironmentStorage, CodegenStorage)] struct Database { storage: salsa::Storage, + fs: StandardFileSystem, } impl salsa::Database for Database {} @@ -54,33 +55,6 @@ impl salsa::Database for Database {} impl FileSystem for Database { fn read(&self, uri: &URI<'_>) -> Result, ReadError> { - let _span = tracing::info_span!("read", %uri).entered(); - - match uri.scheme() { - Scheme::File => { - let filename = uri.path().to_string(); - let contents = - std::fs::read(&filename).map_err(ReadError::other)?; - - tracing::info!(bytes_read = contents.len(), %filename, "Read a file from disk"); - - Ok(contents.into()) - }, - Scheme::HTTP | Scheme::HTTPS => { - tracing::info!("Downloading"); - let response = reqwest::blocking::get(uri.to_string()) - .and_then(|r| r.error_for_status()) - .map_err(ReadError::other)?; - - let body = response.bytes().map_err(ReadError::other)?; - tracing::debug!(bytes_read = body.len(), "Download complete"); - - Ok(body.to_vec().into()) - }, - Scheme::Unregistered(s) if s.as_str() == "wapm" => { - Ok(Vector::default()) - }, - _ => unimplemented!(), - } + self.fs.read(uri) } } diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index 231b3106a3..691b9871c5 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -179,8 +179,10 @@ mod tests { use super::*; use crate::{ - parse::Frontend, parse::FrontendStorage, BuildConfig, Environment, - EnvironmentStorage, FeatureFlags, FileSystem, ReadError, + filesystem::{FileSystem, ReadError}, + parse::Frontend, + parse::FrontendStorage, + BuildConfig, Environment, EnvironmentStorage, FeatureFlags, }; #[derive(Default)] diff --git a/crates/compiler/src/filesystem.rs b/crates/compiler/src/filesystem.rs deleted file mode 100644 index d10468ea32..0000000000 --- a/crates/compiler/src/filesystem.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::{ - fmt::{self, Debug, Display, Formatter}, - sync::Arc, -}; - -use uriparse::URI; - -use crate::{im::Vector, Text}; - -/// An abstract filesystem. -pub trait FileSystem { - /// Read a file's contents from somewhere, with the interpretation changing - /// depending on the URI's scheme. - fn read(&self, path: &URI<'_>) -> Result, ReadError>; -} - -#[derive(Debug, Clone, thiserror::Error)] -pub enum ReadError { - #[error("The \"{}\" scheme isn't supported", scheme)] - UnsupportedScheme { scheme: Text }, - #[error(transparent)] - Other(Arc), -} - -impl ReadError { - pub fn msg(error_message: impl Display + Send + Sync + 'static) -> Self { - struct Message(T); - - impl Display for Message { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } - } - - impl Debug for Message { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_tuple("Message") - .field(&format_args!("{}", self.0)) - .finish() - } - } - - impl std::error::Error for Message {} - - ReadError::other(Message(error_message)) - } - - pub fn other( - error: impl std::error::Error + Send + Sync + 'static, - ) -> Self { - ReadError::Other(Arc::new(error)) - } -} diff --git a/crates/compiler/src/filesystem/builtin.rs b/crates/compiler/src/filesystem/builtin.rs new file mode 100644 index 0000000000..aeee6af3a0 --- /dev/null +++ b/crates/compiler/src/filesystem/builtin.rs @@ -0,0 +1,107 @@ +use std::path::{Component, Path, PathBuf}; + +use reqwest::blocking::Client; +use uriparse::{Scheme, URI}; + +use crate::{ + filesystem::{FileSystem, ReadError, WapmUri}, + im::Vector, +}; + +#[derive(Debug, Clone)] +pub struct StandardFileSystem { + client: Client, + root_directory: PathBuf, +} + +impl StandardFileSystem { + pub fn new(root_directory: impl Into) -> Self { + StandardFileSystem { + client: Client::default(), + root_directory: root_directory.into(), + } + } +} + +impl Default for StandardFileSystem { + fn default() -> Self { + let current_dir = std::env::current_dir().unwrap_or_default(); + StandardFileSystem::new(current_dir) + } +} + +impl FileSystem for StandardFileSystem { + #[tracing::instrument(skip(self), err)] + fn read(&self, uri: &URI<'_>) -> Result, ReadError> { + match uri.scheme() { + Scheme::HTTP | Scheme::HTTPS => { + http_file(&self.client, uri).map_err(ReadError::from) + }, + Scheme::File => local_file(&self.root_directory, uri.path()), + Scheme::Unregistered(u) if u.as_str().is_empty() => { + local_file(&self.root_directory, uri.path()) + }, + Scheme::Unregistered(u) if u.as_str() == "wapm" => { + let uri: WapmUri = uri.try_into().map_err(ReadError::other)?; + todo!() + }, + other => Err(ReadError::UnsupportedScheme { + scheme: other.as_str().into(), + }), + } + } +} + +#[tracing::instrument(skip_all)] +fn http_file( + client: &Client, + url: &URI<'_>, +) -> Result, reqwest::Error> { + tracing::info!(%url, "Downloading"); + let response = client.get(url.to_string()).send()?.error_for_status()?; + + let status_code = response.status(); + + let body = response.bytes()?; + let body = Vector::from(&*body); + + tracing::info!( + status_code = status_code.as_u16(), + status = %status_code, + bytes_read = body.len(), + "Downloaded", + ); + + Ok(body) +} + +#[tracing::instrument(skip_all)] +fn local_file( + root: &Path, + uri: &uriparse::Path<'_>, +) -> Result, ReadError> { + let mut path = if uri.is_absolute() { + PathBuf::from(Component::RootDir.as_os_str()) + } else { + root.to_path_buf() + }; + + for segment in uri.segments() { + path.push(segment.as_str()); + } + + tracing::debug!(path = %path.display(), "Reading a file from disk"); + std::fs::read(&path) + .map(Vector::from) + .map_err(ReadError::from) +} + +impl From for ReadError { + fn from(e: reqwest::Error) -> Self { + if e.status() == Some(reqwest::StatusCode::NOT_FOUND) { + ReadError::NotFound + } else { + ReadError::other(e) + } + } +} diff --git a/crates/compiler/src/filesystem/mod.rs b/crates/compiler/src/filesystem/mod.rs new file mode 100644 index 0000000000..035b978bb3 --- /dev/null +++ b/crates/compiler/src/filesystem/mod.rs @@ -0,0 +1,288 @@ +#[cfg(feature = "builtins")] +mod builtin; + +pub use self::builtin::StandardFileSystem; + +use std::{ + collections::HashMap, + fmt::{self, Debug, Display, Formatter}, + str::FromStr, + sync::{Arc, RwLock}, +}; + +use serde::Deserialize; +use uriparse::{Scheme, URI}; + +use crate::{im::Vector, Text}; + +/// An abstract filesystem. +pub trait FileSystem { + /// Read a file's contents from somewhere, with the interpretation changing + /// depending on the URI's scheme. + fn read(&self, uri: &URI<'_>) -> Result, ReadError>; + + /// Wrap a [`FileSystem`] in a simple caching layer. + fn cached(self) -> Cached + where + Self: Sized, + { + Cached::new(self) + } +} + +#[derive(Debug, Clone, thiserror::Error)] +pub enum ReadError { + #[error("The \"{}\" scheme isn't supported", scheme)] + UnsupportedScheme { scheme: Text }, + #[error("The item wasn't found")] + NotFound, + #[error(transparent)] + Other(Arc), +} + +impl ReadError { + pub fn msg(error_message: impl Display + Send + Sync + 'static) -> Self { + struct Message(T); + + impl Display for Message { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } + } + + impl Debug for Message { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_tuple("Message") + .field(&format_args!("{}", self.0)) + .finish() + } + } + + impl std::error::Error for Message {} + + ReadError::other(Message(error_message)) + } + + pub fn other( + error: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + ReadError::Other(Arc::new(error)) + } +} + +impl From for ReadError { + fn from(e: std::io::Error) -> Self { + if e.kind() == std::io::ErrorKind::NotFound { + ReadError::NotFound + } else { + ReadError::other(e) + } + } +} + +#[derive(Debug)] +pub struct Cached { + fs: F, + cache: RwLock, Vector>>, +} + +impl Cached { + fn new(fs: F) -> Self { + Cached { + fs, + cache: RwLock::new(HashMap::new()), + } + } + + pub fn inner(&self) -> &F { + &self.fs + } + + pub fn into_inner(self) -> F { + let Cached { fs, .. } = self; + fs + } +} + +impl FileSystem for Cached { + #[tracing::instrument(skip(self), err)] + fn read(&self, uri: &URI<'_>) -> Result, ReadError> { + if let Some(cached_value) = + self.cache.read().ok().and_then(|c| c.get(uri).cloned()) + { + tracing::debug!(%uri, bytes = cached_value.len(),"Cache hit!"); + return Ok(cached_value); + } + + let value = self.fs.read(uri)?; + + tracing::debug!(%uri, bytes = value.len(),"Adding entry to cache"); + + self.cache + .write() + .unwrap() + .insert(uri.clone().into_owned(), value.clone()); + + Ok(value) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct WapmUri { + pub namespace: String, + pub package_name: String, + pub version: Option, +} + +impl FromStr for WapmUri { + type Err = ParseWapmUriError; + + fn from_str(s: &str) -> Result { + s.try_into() + } +} + +impl<'a> TryFrom<&'a str> for WapmUri { + type Error = ParseWapmUriError; + + fn try_from(value: &'a str) -> Result { + let uri: URI = value.try_into()?; + WapmUri::try_from(&uri) + } +} + +impl TryFrom<&'_ URI<'_>> for WapmUri { + type Error = ParseWapmUriError; + + fn try_from(uri: &URI<'_>) -> Result { + if uri.scheme().as_str() != "wapm" { + return Err(ParseWapmUriError::IncorrectSchema( + uri.scheme().clone().into_owned(), + )); + } + + let path = uri.path(); + + if !path.is_absolute() { + return Err(ParseWapmUriError::NotAbsolute); + } else if let Some(uriparse::Host::RegisteredName(name)) = + uri.authority().map(|a| a.host()) + { + if !name.is_empty() { + return Err(ParseWapmUriError::NotAbsolute); + } + } + + let (namespace, package_name) = match path.segments() { + [ns, pkg] => (ns.to_string(), pkg.to_string()), + _ => { + return Err(ParseWapmUriError::MalformedPath( + path.clone().into_owned(), + )) + }, + }; + + let version = parse_version_from_query(&uri)?; + + Ok(WapmUri { + namespace, + package_name, + version, + }) + } +} + +fn parse_version_from_query( + uri: &URI<'_>, +) -> Result, ParseWapmUriError> { + let query = match uri.query() { + Some(q) => q, + None => return Ok(None), + }; + + let parsed = queryst::parse(query.as_borrowed().as_str()) + .map_err(|e| ParseWapmUriError::InvalidQueryString(e.message))?; + + #[derive(serde::Deserialize)] + struct Query { + version: Option, + } + + let Query { version } = Query::deserialize(&parsed).map_err(|e| { + ParseWapmUriError::InvalidQueryParameters { + error: e, + query: query.as_str().to_string(), + } + })?; + + Ok(version) +} + +#[derive(Debug, thiserror::Error)] +pub enum ParseWapmUriError { + #[error("Expected a \"wapm\" scheme but found \"{_0}\"")] + IncorrectSchema(Scheme<'static>), + #[error("The URI should be an absolute path (i.e. wapm:///...)")] + NotAbsolute, + #[error("Expected a path like \"hotg-ai/normalize\", but found \"{_0}\"")] + MalformedPath(uriparse::Path<'static>), + #[error("Unable to parse the string as a URI")] + InvalidUri(#[from] uriparse::URIError), + #[error("{_0}")] + InvalidQueryString(String), + #[error("Unable to parse the version from \"{query}\"")] + InvalidQueryParameters { + #[source] + error: serde_json::Error, + query: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn incorrect_schema() { + let uri = "https://example.com"; + + let err = WapmUri::try_from(uri).unwrap_err(); + + assert!(matches!( + err, + ParseWapmUriError::IncorrectSchema(Scheme::HTTPS) + )); + } + + #[test] + fn absolute_path() { + let uri = "wapm://hotg-ai/normalize"; + + let err = WapmUri::try_from(uri).unwrap_err(); + + assert!(matches!(err, ParseWapmUriError::NotAbsolute)); + } + + #[test] + fn path_too_long() { + let uri = "wapm:///path/to/normalize"; + + let err = WapmUri::try_from(uri).unwrap_err(); + + assert!(matches!(err, ParseWapmUriError::MalformedPath(_))); + } + + #[test] + fn parse_wapm_uri() { + let uri = "wapm:///hotg-ai/normalize?version=0.12"; + let should_be = WapmUri { + namespace: "hotg-ai".to_string(), + package_name: "normalize".to_string(), + version: Some("0.12".to_string()), + }; + + let got: WapmUri = uri.parse().unwrap(); + + assert_eq!(got, should_be); + } +} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index ecb21a4ac5..7141d0f121 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -6,14 +6,13 @@ extern crate pretty_assertions; pub mod codegen; mod config; -mod filesystem; +pub mod filesystem; pub mod im; pub mod parse; pub mod type_check; pub use crate::{ config::{BuildConfig, Environment, EnvironmentStorage, FeatureFlags}, - filesystem::{FileSystem, ReadError}, im::Text, }; diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index e03dbee7c6..4986d649ac 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -3,13 +3,14 @@ use std::{collections::BTreeMap, sync::Arc}; use uriparse::{URIBuilder, URIError, URI}; use crate::{ + filesystem::FileSystem, im::{OrdMap, Vector}, parse::{ CapabilityStage, Document, DocumentV1, ItemType, ModelStage, NotFound, ParseFailed, Path, ProcBlockStage, ResourceDeclaration, Stage, WellKnownPath, WrongItemType, }, - BuildConfig, Environment, FileSystem, Text, + BuildConfig, Environment, Text, }; /// The Rune compiler's YAML frontend. @@ -23,7 +24,8 @@ use crate::{ /// ```rust /// use hotg_rune_compiler::{ /// parse::{Frontend, FrontendStorage}, -/// EnvironmentStorage, FileSystem, ReadError, parse::Path, im::Vector, +/// filesystem::{FileSystem, ReadError}, +/// parse::Path, EnvironmentStorage, im::Vector, /// }; /// use uriparse::URI; /// From 00506cae691aff4f5af2d064fe64262f9d528f4a Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Jun 2022 22:32:15 +0800 Subject: [PATCH 61/84] Switched the Rune CLI over to the builtin filesystem --- crates/compiler/src/filesystem/builtin.rs | 2 +- crates/rune-cli/src/build/abi_v1.rs | 98 ++--------------------- 2 files changed, 9 insertions(+), 91 deletions(-) diff --git a/crates/compiler/src/filesystem/builtin.rs b/crates/compiler/src/filesystem/builtin.rs index aeee6af3a0..7e4448379a 100644 --- a/crates/compiler/src/filesystem/builtin.rs +++ b/crates/compiler/src/filesystem/builtin.rs @@ -42,7 +42,7 @@ impl FileSystem for StandardFileSystem { local_file(&self.root_directory, uri.path()) }, Scheme::Unregistered(u) if u.as_str() == "wapm" => { - let uri: WapmUri = uri.try_into().map_err(ReadError::other)?; + let _uri: WapmUri = uri.try_into().map_err(ReadError::other)?; todo!() }, other => Err(ReadError::UnsupportedScheme { diff --git a/crates/rune-cli/src/build/abi_v1.rs b/crates/rune-cli/src/build/abi_v1.rs index 1c24b4eef4..46f8c1c7a5 100644 --- a/crates/rune-cli/src/build/abi_v1.rs +++ b/crates/rune-cli/src/build/abi_v1.rs @@ -1,16 +1,15 @@ -use std::{convert::TryInto, fmt::Display, path::PathBuf, sync::Arc}; +use std::path::PathBuf; use anyhow::{Context, Error}; use query_based_compiler::{ codegen::{Codegen, CodegenStorage}, + filesystem::{FileSystem, ReadError, StandardFileSystem}, im::Vector, parse::{Frontend, FrontendStorage}, - BuildConfig, Environment, EnvironmentStorage, FeatureFlags, FileSystem, - ReadError, + BuildConfig, Environment, EnvironmentStorage, FeatureFlags, }; use salsa::Storage; -use serde::Deserialize; -use uriparse::{Scheme, URI}; +use uriparse::URI; use crate::{Build, Unstable}; @@ -28,6 +27,7 @@ pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { let mut db = Database { storage: Storage::default(), current_dir: build.current_directory()?, + fs: StandardFileSystem::default(), }; db.set_config(BuildConfig { @@ -53,95 +53,13 @@ pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { struct Database { storage: Storage, current_dir: PathBuf, + fs: StandardFileSystem, } impl salsa::Database for Database {} impl FileSystem for Database { - fn read(&self, path: &URI<'_>) -> Result, ReadError> { - match path.scheme() { - Scheme::FileSystem | Scheme::File => read_file(path.path()), - Scheme::HTTP | Scheme::HTTPS => download_from_the_internet(path), - Scheme::Unregistered(u) if u.as_str().is_empty() => { - read_file(path.path()) - }, - Scheme::Unregistered(u) if u.as_str() == "wapm" => { - download_from_wapm(path) - }, - other => Err(ReadError::UnsupportedScheme { - scheme: other.as_str().into(), - }), - } + fn read(&self, uri: &URI<'_>) -> Result, ReadError> { + self.fs.read(uri) } } - -fn read_file(path: &uriparse::Path<'_>) -> Result, ReadError> { - let mut full_path = PathBuf::new(); - - if path.is_absolute() { - full_path.push(std::path::Component::RootDir); - } - - for segment in path.segments() { - full_path.push(segment.as_str()); - } - - std::fs::read(&full_path) - .map(Vector::from) - .map_err(|e| ReadError::Other(Arc::new(e) as Arc<_>)) -} - -#[tracing::instrument] -fn download_from_wapm(uri: &URI<'_>) -> Result, ReadError> { - let (namespace, package_name) = match uri.path().segments() { - [ns, pkg] => (ns.as_str(), pkg.as_str()), - _ => { - return Err(ReadError::other(MalformedPackagePath { - path: uri.path().clone().into_owned(), - })) - }, - }; - - // https://registry-cdn.wapm.io/contents/hotg-ai/softmax/0.12.0/softmax.wasm - let query = uri.query().map(|q| q.as_str()).unwrap_or_default(); - let query = queryst::parse(query).map_err(|e| ReadError::msg(e.message))?; - let query_params: QueryParams = - QueryParams::deserialize(&query).map_err(ReadError::other)?; - let version = query_params - .version - .ok_or_else(|| ReadError::msg("No version specified"))?; - - let wapm_url = format!("https://registry-cdn.wapm.io/contents/{namespace}/{package_name}/{version}/{package_name}.wasm"); - let wapm_url = wapm_url.as_str().try_into().map_err(ReadError::other)?; - - download_from_the_internet(&wapm_url) -} - -#[tracing::instrument] -fn download_from_the_internet(uri: &URI<'_>) -> Result, ReadError> { - let url = uri.to_string(); - let body = reqwest::blocking::get(&url) - .and_then(|response| response.error_for_status()) - .and_then(|response| response.bytes()) - .map_err(ReadError::other)?; - - Ok(body.as_ref().into()) -} - -#[derive(Debug, serde::Deserialize)] -struct QueryParams { - version: Option, -} - -#[derive(Debug)] -struct MalformedPackagePath { - path: uriparse::Path<'static>, -} - -impl Display for MalformedPackagePath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Unable to determine the package name and namespace from \"{}\". Expected something like /", self.path) - } -} - -impl std::error::Error for MalformedPackagePath {} From 6ce26843a80d221023be2bdaf820c330cb3fc0bf Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Jun 2022 23:39:23 +0800 Subject: [PATCH 62/84] Adding docs and fleshing out the WapmUri type --- crates/compiler/examples/compile.rs | 6 +- .../{filesystem => asset_loader}/builtin.rs | 15 +- .../src/{filesystem => asset_loader}/mod.rs | 151 +++++++++++++++--- crates/compiler/src/codegen/query.rs | 4 +- crates/compiler/src/lib.rs | 2 +- crates/compiler/src/parse/query.rs | 13 +- crates/rune-cli/src/build/abi_v1.rs | 8 +- 7 files changed, 157 insertions(+), 42 deletions(-) rename crates/compiler/src/{filesystem => asset_loader}/builtin.rs (88%) rename crates/compiler/src/{filesystem => asset_loader}/mod.rs (60%) diff --git a/crates/compiler/examples/compile.rs b/crates/compiler/examples/compile.rs index b9298a7300..3e79f21d9c 100644 --- a/crates/compiler/examples/compile.rs +++ b/crates/compiler/examples/compile.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use hotg_rune_compiler::{ codegen::{Codegen, CodegenStorage}, - filesystem::{FileSystem, ReadError, StandardFileSystem}, + asset_loader::{AssetLoader, DefaultAssetLoader, ReadError}, im::Vector, parse::{Frontend, FrontendStorage}, BuildConfig, Environment, EnvironmentStorage, FeatureFlags, @@ -45,7 +45,7 @@ fn main() { #[salsa::database(FrontendStorage, EnvironmentStorage, CodegenStorage)] struct Database { storage: salsa::Storage, - fs: StandardFileSystem, + fs: DefaultAssetLoader, } impl salsa::Database for Database {} @@ -53,7 +53,7 @@ impl salsa::Database for Database {} // The parsing process requires you to load proc-blocks and read files. You // can satisfy these dependencies by implementing the corresponding traits. -impl FileSystem for Database { +impl AssetLoader for Database { fn read(&self, uri: &URI<'_>) -> Result, ReadError> { self.fs.read(uri) } diff --git a/crates/compiler/src/filesystem/builtin.rs b/crates/compiler/src/asset_loader/builtin.rs similarity index 88% rename from crates/compiler/src/filesystem/builtin.rs rename to crates/compiler/src/asset_loader/builtin.rs index 7e4448379a..6550fc7322 100644 --- a/crates/compiler/src/filesystem/builtin.rs +++ b/crates/compiler/src/asset_loader/builtin.rs @@ -4,33 +4,34 @@ use reqwest::blocking::Client; use uriparse::{Scheme, URI}; use crate::{ - filesystem::{FileSystem, ReadError, WapmUri}, + asset_loader::{AssetLoader, ReadError, WapmUri}, im::Vector, }; +/// A [`FileLoader`] that uses a "pretty good" strategy for retrieving assets. #[derive(Debug, Clone)] -pub struct StandardFileSystem { +pub struct DefaultAssetLoader { client: Client, root_directory: PathBuf, } -impl StandardFileSystem { +impl DefaultAssetLoader { pub fn new(root_directory: impl Into) -> Self { - StandardFileSystem { + DefaultAssetLoader { client: Client::default(), root_directory: root_directory.into(), } } } -impl Default for StandardFileSystem { +impl Default for DefaultAssetLoader { fn default() -> Self { let current_dir = std::env::current_dir().unwrap_or_default(); - StandardFileSystem::new(current_dir) + DefaultAssetLoader::new(current_dir) } } -impl FileSystem for StandardFileSystem { +impl AssetLoader for DefaultAssetLoader { #[tracing::instrument(skip(self), err)] fn read(&self, uri: &URI<'_>) -> Result, ReadError> { match uri.scheme() { diff --git a/crates/compiler/src/filesystem/mod.rs b/crates/compiler/src/asset_loader/mod.rs similarity index 60% rename from crates/compiler/src/filesystem/mod.rs rename to crates/compiler/src/asset_loader/mod.rs index 035b978bb3..96fa0011dd 100644 --- a/crates/compiler/src/filesystem/mod.rs +++ b/crates/compiler/src/asset_loader/mod.rs @@ -1,7 +1,8 @@ #[cfg(feature = "builtins")] mod builtin; -pub use self::builtin::StandardFileSystem; +#[cfg(feature = "builtins")] +pub use self::builtin::DefaultAssetLoader; use std::{ collections::HashMap, @@ -15,13 +16,19 @@ use uriparse::{Scheme, URI}; use crate::{im::Vector, Text}; -/// An abstract filesystem. -pub trait FileSystem { - /// Read a file's contents from somewhere, with the interpretation changing - /// depending on the URI's scheme. +/// Something that can load external resources using a URI. +/// +/// +/// - **`http` and `https`**: Make a GET request and use the response body, +/// erroring out if the server doesn't send back a 2XX response +/// - **file**: Read a file from disk +/// - **wapm**: Retrieve a WebAssembly module from the WebAssembly Package +/// Manager (see also [`WapmUri`]) +pub trait AssetLoader { + /// Fetch an asset using some URI. fn read(&self, uri: &URI<'_>) -> Result, ReadError>; - /// Wrap a [`FileSystem`] in a simple caching layer. + /// Wrap a [`AssetLoader`] in a simple caching layer. fn cached(self) -> Cached where Self: Sized, @@ -104,7 +111,7 @@ impl Cached { } } -impl FileSystem for Cached { +impl AssetLoader for Cached { #[tracing::instrument(skip(self), err)] fn read(&self, uri: &URI<'_>) -> Result, ReadError> { if let Some(cached_value) = @@ -127,11 +134,82 @@ impl FileSystem for Cached { } } +/// A URI which represents a WebAssembly module on WAPM. +/// +/// Some examples are: +/// - The simplest WAPM URI - `wapm:///hotg-ai/normalize` +/// - Use a particular Semver version - `wapm:///hotg-ai/normalize?version=0.12` +/// - Use the `first` module from latest version of `hotg-ai/normalize` - +/// `wapm:///hotg-ai/normalize?version=latest&module=first` #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct WapmUri { pub namespace: String, pub package_name: String, pub version: Option, + pub module: Option, +} + +impl WapmUri { + pub fn new( + namespace: impl Into, + package_name: impl Into, + ) -> Self { + WapmUri { + namespace: namespace.into(), + package_name: package_name.into(), + version: None, + module: None, + } + } + + pub fn with_version(self, version: impl Into) -> Self { + WapmUri { + version: Some(version.into()), + ..self + } + } + + pub fn with_module(self, module: impl Into) -> Self { + WapmUri { + module: Some(module.into()), + ..self + } + } + + pub fn as_uri(&self) -> Result, uriparse::URIError> { + let WapmUri { + namespace, + package_name, + version, + module, + } = self; + let mut path: uriparse::Path = namespace.as_str().try_into()?; + path.push(package_name.as_str())?; + path.set_absolute(true); + + let query = match (version.as_deref(), module.as_deref()) { + (Some(v), Some(m)) => Some(format!("version={v}&module={m}")), + (Some(v), None) => Some(format!("version={v}")), + (None, Some(m)) => Some(format!("module={m}")), + (None, None) => None, + }; + let query = query + .as_deref() + .and_then(|q| uriparse::Query::try_from(q).ok()) + .map(|q| q.into_owned()); + + let mut builder = URI::builder().with_path(path).with_query(query); + builder.try_scheme("wapm")?.try_authority(Some(""))?; + + builder.build() + } +} + +impl Display for WapmUri { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let uri = self.as_uri().map_err(|_| fmt::Error)?; + Display::fmt(&uri, f) + } } impl FromStr for WapmUri { @@ -182,40 +260,43 @@ impl TryFrom<&'_ URI<'_>> for WapmUri { }, }; - let version = parse_version_from_query(&uri)?; + let WapmQueryParameters { version, module } = + parse_version_from_query(&uri)?; Ok(WapmUri { namespace, package_name, version, + module, }) } } +#[derive(Default, serde::Deserialize)] +struct WapmQueryParameters { + version: Option, + module: Option, +} + fn parse_version_from_query( uri: &URI<'_>, -) -> Result, ParseWapmUriError> { +) -> Result { let query = match uri.query() { Some(q) => q, - None => return Ok(None), + None => return Ok(WapmQueryParameters::default()), }; let parsed = queryst::parse(query.as_borrowed().as_str()) .map_err(|e| ParseWapmUriError::InvalidQueryString(e.message))?; - #[derive(serde::Deserialize)] - struct Query { - version: Option, - } - - let Query { version } = Query::deserialize(&parsed).map_err(|e| { + let params = WapmQueryParameters::deserialize(&parsed).map_err(|e| { ParseWapmUriError::InvalidQueryParameters { error: e, query: query.as_str().to_string(), } })?; - Ok(version) + Ok(params) } #[derive(Debug, thiserror::Error)] @@ -273,16 +354,48 @@ mod tests { } #[test] - fn parse_wapm_uri() { - let uri = "wapm:///hotg-ai/normalize?version=0.12"; + fn parse_simple_wapm_uri() { + let uri = "wapm:///hotg-ai/normalize"; + let should_be = WapmUri { + namespace: "hotg-ai".to_string(), + package_name: "normalize".to_string(), + version: None, + module: None, + }; + + let got: WapmUri = uri.parse().unwrap(); + + assert_eq!(got, should_be); + } + + #[test] + fn parse_full_wapm_uri() { + let uri = "wapm:///hotg-ai/normalize?version=0.12&module=first"; let should_be = WapmUri { namespace: "hotg-ai".to_string(), package_name: "normalize".to_string(), version: Some("0.12".to_string()), + module: Some("first".to_string()), }; let got: WapmUri = uri.parse().unwrap(); assert_eq!(got, should_be); } + + #[test] + fn round_trip_wapm_uris() { + let uris = [ + "wapm:///hotg-ai/normalize", + "wapm:///hotg-ai/normalize?module=first", + "wapm:///hotg-ai/normalize?version=0.12", + "wapm:///hotg-ai/normalize?version=0.12&module=first", + ]; + + for uri in uris { + let parsed: WapmUri = uri.parse().unwrap(); + let round_tripped = parsed.to_string(); + assert_eq!(round_tripped, uri); + } + } } diff --git a/crates/compiler/src/codegen/query.rs b/crates/compiler/src/codegen/query.rs index 691b9871c5..71b88a92ea 100644 --- a/crates/compiler/src/codegen/query.rs +++ b/crates/compiler/src/codegen/query.rs @@ -179,7 +179,7 @@ mod tests { use super::*; use crate::{ - filesystem::{FileSystem, ReadError}, + asset_loader::{AssetLoader, ReadError}, parse::Frontend, parse::FrontendStorage, BuildConfig, Environment, EnvironmentStorage, FeatureFlags, @@ -193,7 +193,7 @@ mod tests { impl salsa::Database for Database {} - impl FileSystem for Database { + impl AssetLoader for Database { fn read(&self, path: &URI<'_>) -> Result, ReadError> { // Note: The tests don't actually care about the value we get back. match path.scheme() { diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 7141d0f121..c567be98cc 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -6,7 +6,7 @@ extern crate pretty_assertions; pub mod codegen; mod config; -pub mod filesystem; +pub mod asset_loader; pub mod im; pub mod parse; pub mod type_check; diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index 4986d649ac..299973ca1a 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, sync::Arc}; use uriparse::{URIBuilder, URIError, URI}; use crate::{ - filesystem::FileSystem, + asset_loader::AssetLoader, im::{OrdMap, Vector}, parse::{ CapabilityStage, Document, DocumentV1, ItemType, ModelStage, NotFound, @@ -24,7 +24,7 @@ use crate::{ /// ```rust /// use hotg_rune_compiler::{ /// parse::{Frontend, FrontendStorage}, -/// filesystem::{FileSystem, ReadError}, +/// asset_loader::{AssetLoader, ReadError}, /// parse::Path, EnvironmentStorage, im::Vector, /// }; /// use uriparse::URI; @@ -39,10 +39,11 @@ use crate::{ /// /// impl salsa::Database for Database {} /// -/// // The parsing process requires you to load proc-blocks and read files. You -/// // can satisfy these dependencies by implementing the corresponding traits. +/// // The parsing process requires you to load proc-blocks and other assets. +/// // You can satisfy these dependencies by implementing the corresponding +/// // traits. /// -/// impl FileSystem for Database { +/// impl AssetLoader for Database { /// fn read(&self, path: &URI<'_>) -> Result, ReadError> { /// todo!(); /// } @@ -70,7 +71,7 @@ use crate::{ /// let doc = db.parse().unwrap(); /// ``` #[salsa::query_group(FrontendStorage)] -pub trait Frontend: Environment + FileSystem { +pub trait Frontend: Environment + AssetLoader { /// The YAML document being parsed. #[salsa::input] fn src(&self) -> Text; diff --git a/crates/rune-cli/src/build/abi_v1.rs b/crates/rune-cli/src/build/abi_v1.rs index 46f8c1c7a5..9d51cd0a5c 100644 --- a/crates/rune-cli/src/build/abi_v1.rs +++ b/crates/rune-cli/src/build/abi_v1.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use anyhow::{Context, Error}; use query_based_compiler::{ codegen::{Codegen, CodegenStorage}, - filesystem::{FileSystem, ReadError, StandardFileSystem}, + asset_loader::{AssetLoader, DefaultAssetLoader, ReadError}, im::Vector, parse::{Frontend, FrontendStorage}, BuildConfig, Environment, EnvironmentStorage, FeatureFlags, @@ -27,7 +27,7 @@ pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { let mut db = Database { storage: Storage::default(), current_dir: build.current_directory()?, - fs: StandardFileSystem::default(), + fs: DefaultAssetLoader::default(), }; db.set_config(BuildConfig { @@ -53,12 +53,12 @@ pub(crate) fn execute(build: Build, unstable: Unstable) -> Result<(), Error> { struct Database { storage: Storage, current_dir: PathBuf, - fs: StandardFileSystem, + fs: DefaultAssetLoader, } impl salsa::Database for Database {} -impl FileSystem for Database { +impl AssetLoader for Database { fn read(&self, uri: &URI<'_>) -> Result, ReadError> { self.fs.read(uri) } From 95e16dee0871fa35273871c7d4fcef4b7c82b455 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 3 Jun 2022 01:35:07 +0800 Subject: [PATCH 63/84] Updated a doc-comment --- crates/compiler/src/asset_loader/builtin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compiler/src/asset_loader/builtin.rs b/crates/compiler/src/asset_loader/builtin.rs index 6550fc7322..b99bb6c53a 100644 --- a/crates/compiler/src/asset_loader/builtin.rs +++ b/crates/compiler/src/asset_loader/builtin.rs @@ -8,7 +8,7 @@ use crate::{ im::Vector, }; -/// A [`FileLoader`] that uses a "pretty good" strategy for retrieving assets. +/// An [`AssetLoader`] that loads assets using the basic solution. #[derive(Debug, Clone)] pub struct DefaultAssetLoader { client: Client, From 135f641dc3fbb5407055d719afc3b5e7256ba010 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 3 Jun 2022 02:28:37 +0800 Subject: [PATCH 64/84] Switch the git submodule url to HTTPS instead of SSH so cargo can clone it --- .gitmodules | 2 +- crates/runtime/Cargo.toml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 63b2e8eb16..74b33f74e2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "wit-files"] path = wit-files - url = git@github.com:hotg-ai/wit-files.git + url = https://github.com/hotg-ai/wit-files.git diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index ac1e8f2bcb..d4cd054168 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -78,7 +78,5 @@ tflite = ["hotg-runecoral"] unstable_doc_cfg = [] zune = ["wasmer", "zip"] -[metadata] -[metadata.docs] -[metadata.docs.rs] +[package.metadata.docs.rs] all-features = true From 60b3d1d3a04bb7261d0c1dffb3eb781b193a2ab0 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 3 Jun 2022 07:23:47 +0800 Subject: [PATCH 65/84] Cleaned up the Cargo.toml file --- crates/runtime/Cargo.toml | 69 ++++++++++----------------------------- 1 file changed, 17 insertions(+), 52 deletions(-) diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index d4cd054168..c3d0925986 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -10,70 +10,35 @@ name = "hotg-rune-runtime" readme = "README.md" repository = "https://github.com/hotg-ai/rune" version = "0.11.3" + [dependencies] anyhow = "1.0.40" +csv = { version = "1.1.6", optional = true } +hotg-rune-compiler = { version = "^0.11.0", path = "../compiler" } +hotg-rune-core = { version = "^0.11.0", path = "../rune-core", features = ["std"] } +hotg-runecoral = { version = "0.3.12", optional = true } +hound = { version = "3.4.0", optional = true } +image = { version = "0.23.14", optional = true } +indexmap = { version = "1.8.0", features = ["serde-1"] } log = "0.4.14" +rand = { version = "0.8.3", optional = true } +serde = { version = "1.0.136", features = ["derive"] } +serde_json = "1.0.79" thiserror = "1.0.30" -wasmparser = "0.83.0" -wit-bindgen-wasmer = { git = "https://github.com/wasmerio/wit-bindgen", branch = "wasmer" } tracing = { version = "0.1.33", features = ["attributes"] } tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } -indexmap = { version = "1.8.0", features = ["serde-1"] } - -[dependencies.csv] -optional = true -version = "1.1.6" - -[dependencies.hotg-rune-compiler] -path = "../compiler" -version = "^0.11.0" - -[dependencies.hotg-rune-core] -features = ["std"] -path = "../rune-core" -version = "^0.11.0" - -[dependencies.hotg-runecoral] -optional = true -version = "0.3.12" - -[dependencies.hound] -optional = true -version = "3.4.0" - -[dependencies.image] -optional = true -version = "0.23.14" - -[dependencies.rand] -optional = true -version = "0.8.3" - -[dependencies.serde] -features = ["derive"] -version = "1.0.136" - -[dependencies.serde_json] -version = "1.0.79" - -[dependencies.wasm3] -git = "https://github.com/wasm3/wasm3-rs" -optional = true - -[dependencies.wasmer] -optional = true -version = "2.2.0-rc2" - -[dependencies.zip] -optional = true -version = "0.6.2" +wasm3 = { git = "https://github.com/wasm3/wasm3-rs", optional = true } +wasmer = { version = "2.2.0-rc2", optional = true } +wasmparser = "0.83.0" +wit-bindgen-wasmer = { git = "https://github.com/wasmerio/wit-bindgen", branch = "wasmer" } +zip = { version = "0.6.2", optional = true } [dev-dependencies] tempfile = "3.2.0" [features] -builtins = ["hound", "image", "rand", "rand/small_rng", "csv"] default = ["builtins", "tflite", "zune"] +builtins = ["hound", "image", "rand", "rand/small_rng", "csv"] tflite = ["hotg-runecoral"] unstable_doc_cfg = [] zune = ["wasmer", "zip"] From 03feefe44d070677c60af59f9c365af63dc5b3f4 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 3 Jun 2022 08:01:35 +0800 Subject: [PATCH 66/84] Sprinkled the Zune code with tracing calls --- Cargo.lock | 2 - crates/runtime/Cargo.toml | 2 - crates/runtime/src/zune.rs | 87 +++++++++++++++++++++++--------------- 3 files changed, 52 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e920a7bef..49fc20bb53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1494,14 +1494,12 @@ dependencies = [ "hound", "image", "indexmap", - "log", "rand 0.8.5", "serde", "serde_json", "tempfile", "thiserror", "tracing", - "tracing-subscriber", "wasm3", "wasmer", "wasmparser 0.83.0", diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index c3d0925986..3f2bab4ee5 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -20,13 +20,11 @@ hotg-runecoral = { version = "0.3.12", optional = true } hound = { version = "3.4.0", optional = true } image = { version = "0.23.14", optional = true } indexmap = { version = "1.8.0", features = ["serde-1"] } -log = "0.4.14" rand = { version = "0.8.3", optional = true } serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.79" thiserror = "1.0.30" tracing = { version = "0.1.33", features = ["attributes"] } -tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } wasm3 = { git = "https://github.com/wasm3/wasm3-rs", optional = true } wasmer = { version = "2.2.0-rc2", optional = true } wasmparser = "0.83.0" diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 4a49dccb14..205f8ede3b 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -61,6 +61,7 @@ pub struct ZuneEngine { } impl ZuneEngine { + #[tracing::instrument(skip_all)] pub fn load(binary: &[u8]) -> Result where Self: Sized, @@ -84,6 +85,8 @@ impl ZuneEngine { let runefile = String::from_utf8(read_zip_resource_by_path("Runefile.yml")?) .context("Unable to read Runefile")?; + tracing::debug!(length=runefile.len(), "Read the Rune"); + let parsed_runefile = Document::parse(&runefile).context("Unable to parse Runefile")?; let pipeline = &parsed_runefile.to_v1().pipeline; @@ -124,8 +127,7 @@ impl ZuneEngine { let tensor_constraints = tensors.iter().map(|_| None).collect(); let shared_state = Arc::new(Mutex::new(State { tensors, tensor_constraints, graph_contexts })); - println!("input_tensors {:?} ", &input_tensors); - println!("output_tensors {:?} ", &output_tensors); + tracing::trace!(?input_tensors, ?output_tensors, "Loaded tensors"); let (model_contexts, procblock_contexts) = instantiate_nodes( pipeline, @@ -136,7 +138,7 @@ impl ZuneEngine { ) .map_err(LoadError::Other)?; - println!(" execution order: {:?}", processing_order); + tracing::debug!(order=?processing_order, "Determined the execution order"); // TODO: Validate and allocate input/output tensors @@ -151,8 +153,11 @@ impl ZuneEngine { }) } + #[tracing::instrument(skip_all)] pub fn predict(&mut self) -> Result<(), Error> { for stage_name in &self.processing_order { + let _span = tracing::debug_span!("Running Stage", %stage_name).entered(); + let stage = self.pipeline.get(stage_name).unwrap(); match stage { Stage::Model(_) => { @@ -263,6 +268,7 @@ impl ZuneEngine { } impl ModelNode { + #[tracing::instrument(skip(node_data, model_data, shared_state, input_tensors, output_tensors), level = "debug")] fn load( node_id: &str, node_data: &ModelStage, @@ -395,6 +401,7 @@ impl ModelNode { }) } + #[tracing::instrument(skip_all, level = "debug")] fn run(&mut self) -> Result<(), Error> { // We are recreating the input_tensors and output_tensors every time // before predict because wasm linear memory might have changed @@ -446,6 +453,7 @@ impl ModelNode { } impl ProcBlockNode { + #[tracing::instrument(skip_all, level = "debug", fields(%node_id))] fn load( node_id: &str, wasm: &[u8], @@ -497,6 +505,7 @@ impl ProcBlockNode { }) } + #[tracing::instrument(skip_all, level = "debug")] fn run(&mut self) -> Result<(), Error> { println!("Executing proc block: {:?} ", self.node_id); // impl stderr for KernelError @@ -907,51 +916,54 @@ impl runtime_v1::RuntimeV1 for Runtime { fn register_node(&mut self, _metadata: &Self::Metadata) { todo!() } + #[tracing::instrument(skip_all, level = "debug")] fn graph_context_for_node( &mut self, - _node_id: &str, + node_id: &str, ) -> Option { self.shared_state .lock() .unwrap() .graph_contexts - .get(_node_id)?; + .get(node_id)?; - Some(_node_id.to_string()) + Some(node_id.to_string()) } + #[tracing::instrument(skip(self, ctx), level = "debug")] fn graph_context_get_argument( &mut self, - _self_: &Self::GraphContext, - _name: &str, + ctx: &Self::GraphContext, + name: &str, ) -> Option { self.shared_state .lock() .unwrap() .graph_contexts - .get(_self_) - .and_then(|c| c.arguments.get(_name).and_then(|v| Some(v.clone()) )) + .get(ctx) + .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()) )) } + #[tracing::instrument(skip(self, ctx), level = "debug")] fn graph_context_add_input_tensor( &mut self, - _self_: &Self::GraphContext, - _name: &str, - _element_type: ElementType, - _dimensions: DimensionsParam<'_>, + ctx: &Self::GraphContext, + name: &str, + element_type: ElementType, + dimensions: DimensionsParam<'_>, ) { self.shared_state .lock() .unwrap() .graph_contexts - .get_mut(_self_) + .get_mut(ctx) .and_then(|c| { c.input_tensors.insert( - _name.to_string(), + name.to_string(), TensorConstraint { tensor_id: None, - element_type: _element_type, - dimensions: match _dimensions { + element_type, + dimensions: match dimensions { DimensionsParam::Dynamic => Dimensions::Dynamic, DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) } @@ -959,25 +971,26 @@ impl runtime_v1::RuntimeV1 for Runtime { }); } + #[tracing::instrument(skip(self, ctx), level = "debug")] fn graph_context_add_output_tensor( &mut self, - _self_: &Self::GraphContext, - _name: &str, - _element_type: ElementType, - _dimensions: DimensionsParam<'_>, + ctx: &Self::GraphContext, + name: &str, + element_type: ElementType, + dimensions: DimensionsParam<'_>, ) { self.shared_state .lock() .unwrap() .graph_contexts - .get_mut(_self_) + .get_mut(ctx) .and_then(|c| { c.output_tensors.insert( - _name.to_string(), + name.to_string(), TensorConstraint { tensor_id: None, - element_type: _element_type, - dimensions: match _dimensions { + element_type: element_type, + dimensions: match dimensions { DimensionsParam::Dynamic => Dimensions::Dynamic, DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) } @@ -985,38 +998,41 @@ impl runtime_v1::RuntimeV1 for Runtime { }); } + #[tracing::instrument(skip_all, level = "debug")] fn kernel_context_for_node( &mut self, - _node_id: &str, + node_id: &str, ) -> Option { self.shared_state.lock() .unwrap() .graph_contexts - .get(_node_id)?; - Some(_node_id.to_string()) + .get(node_id)?; + Some(node_id.to_string()) } + #[tracing::instrument(skip(self, ctx), level = "debug")] fn kernel_context_get_argument( &mut self, - _self_: &Self::KernelContext, + ctx: &Self::KernelContext, name: &str, ) -> Option { self.shared_state .lock() .unwrap() .graph_contexts - .get(_self_) + .get(ctx) .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()) )) } + #[tracing::instrument(skip(self, ctx), level = "debug")] fn kernel_context_get_input_tensor( &mut self, - _self_: &Self::KernelContext, + ctx: &Self::KernelContext, name: &str, ) -> Option { let state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(_self_) + let tensor_id = state.graph_contexts.get(ctx) .and_then(|c| { c.input_tensors .get(name) @@ -1029,9 +1045,10 @@ impl runtime_v1::RuntimeV1 for Runtime { } } + #[tracing::instrument(skip(self, ctx, buffer), level = "debug")] fn kernel_context_set_output_tensor( &mut self, - _self_: &Self::KernelContext, + ctx: &Self::KernelContext, name: &str, TensorParam { element_type, @@ -1041,7 +1058,7 @@ impl runtime_v1::RuntimeV1 for Runtime { ) { let mut state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(_self_) + let tensor_id = state.graph_contexts.get(ctx) .and_then(|c| { c.output_tensors .get(name) From d3198c1ab3ac7fa46284ac3d54bf4cea4f21a90d Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 3 Jun 2022 08:12:45 +0800 Subject: [PATCH 67/84] Fixed some unused variable warnings --- Cargo.lock | 1 + crates/runtime/Cargo.toml | 3 ++- crates/runtime/src/zune.rs | 46 +++++++++++++++++++------------------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49fc20bb53..4775d3116b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1494,6 +1494,7 @@ dependencies = [ "hound", "image", "indexmap", + "log", "rand 0.8.5", "serde", "serde_json", diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 3f2bab4ee5..56192eb140 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -20,11 +20,12 @@ hotg-runecoral = { version = "0.3.12", optional = true } hound = { version = "3.4.0", optional = true } image = { version = "0.23.14", optional = true } indexmap = { version = "1.8.0", features = ["serde-1"] } +log = "0.4.17" rand = { version = "0.8.3", optional = true } serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.79" thiserror = "1.0.30" -tracing = { version = "0.1.33", features = ["attributes"] } +tracing = { version = "0.1.33", features = ["attributes", "log"] } wasm3 = { git = "https://github.com/wasm3/wasm3-rs", optional = true } wasmer = { version = "2.2.0-rc2", optional = true } wasmparser = "0.83.0" diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 205f8ede3b..7c20d315e4 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -361,7 +361,7 @@ impl ModelNode { }, Some(_) => {}, ref mut other => { - other.insert(model_tensor); + *other = Some(model_tensor); }, } @@ -788,27 +788,27 @@ impl runtime_v1::RuntimeV1 for Runtime { fn metadata_set_description( &mut self, - _self_: &Self::Metadata, + _ctx: &Self::Metadata, _description: &str, ) { todo!() } - fn metadata_set_repository(&mut self, _self_: &Self::Metadata, _url: &str) { + fn metadata_set_repository(&mut self, _ctx: &Self::Metadata, _url: &str) { todo!() } - fn metadata_set_homepage(&mut self, _self_: &Self::Metadata, _url: &str) { + fn metadata_set_homepage(&mut self, _ctx: &Self::Metadata, _url: &str) { todo!() } - fn metadata_add_tag(&mut self, _self_: &Self::Metadata, _tag: &str) { + fn metadata_add_tag(&mut self, _ctx: &Self::Metadata, _tag: &str) { todo!() } fn metadata_add_argument( &mut self, - _self_: &Self::Metadata, + _ctx: &Self::Metadata, _arg: &Self::ArgumentMetadata, ) { todo!() @@ -816,7 +816,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn metadata_add_input( &mut self, - _self_: &Self::Metadata, + _ctx: &Self::Metadata, _metadata: &Self::TensorMetadata, ) { todo!() @@ -824,7 +824,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn metadata_add_output( &mut self, - _self_: &Self::Metadata, + _ctx: &Self::Metadata, _metadata: &Self::TensorMetadata, ) { todo!() @@ -836,7 +836,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn argument_metadata_set_description( &mut self, - _self_: &Self::ArgumentMetadata, + _ctx: &Self::ArgumentMetadata, _description: &str, ) { todo!() @@ -844,7 +844,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn argument_metadata_set_default_value( &mut self, - _self_: &Self::ArgumentMetadata, + _ctx: &Self::ArgumentMetadata, _default_value: &str, ) { todo!() @@ -852,7 +852,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn argument_metadata_add_hint( &mut self, - _self_: &Self::ArgumentMetadata, + _ctx: &Self::ArgumentMetadata, _hint: &Self::ArgumentHint, ) { todo!() @@ -864,7 +864,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn tensor_metadata_set_description( &mut self, - _self_: &Self::TensorMetadata, + _ctx: &Self::TensorMetadata, _description: &str, ) { todo!() @@ -872,7 +872,7 @@ impl runtime_v1::RuntimeV1 for Runtime { fn tensor_metadata_add_hint( &mut self, - _self_: &Self::TensorMetadata, + _ctx: &Self::TensorMetadata, _hint: &Self::TensorHint, ) { todo!() @@ -1114,15 +1114,15 @@ impl runtime_v1::RuntimeV1 for Runtime { fn kernel_context_get_global_input( &mut self, - _self_: &Self::KernelContext, - name: &str, + _ctx: &Self::KernelContext, + _name: &str, ) -> Option { todo!() } fn kernel_context_set_global_output( &mut self, - _self_: &Self::KernelContext, + _ctx: &Self::KernelContext, _name: &str, _tensor: TensorParam<'_>, ) { @@ -1131,25 +1131,25 @@ impl runtime_v1::RuntimeV1 for Runtime { fn model_load( &mut self, - model_format: &str, - model: &[u8], - arguments: Vec<(&str, &str)>, + _: &str, + _: &[u8], + _: Vec<(&str, &str)>, ) -> Result { todo!() } - fn model_inputs(&mut self, self_: &Self::Model) -> Vec { + fn model_inputs(&mut self, _: &Self::Model) -> Vec { todo!() } - fn model_outputs(&mut self, self_: &Self::Model) -> Vec { + fn model_outputs(&mut self, _: &Self::Model) -> Vec { todo!() } fn model_infer( &mut self, - self_: &Self::Model, - inputs: Vec>, + _: &Self::Model, + _: Vec>, ) -> Result, ModelInferError> { todo!() } From 467bce637c1cbf560dc5ee56b8a26609d573b36e Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 3 Jun 2022 08:15:47 +0800 Subject: [PATCH 68/84] Ran cargo fmt --- crates/runtime/src/lib.rs | 2 +- crates/runtime/src/zune.rs | 388 ++++++++++++++++++++++++------------- 2 files changed, 258 insertions(+), 132 deletions(-) diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 3fd6515f57..8fe1e6f682 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -39,5 +39,5 @@ pub use crate::{ engine::LoadError, outputs::OutputTensor, runtime::Runtime, - tensor::{ElementType, Tensor, TensorElement} + tensor::{ElementType, Tensor, TensorElement}, }; diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs index 7c20d315e4..9141bd32ac 100644 --- a/crates/runtime/src/zune.rs +++ b/crates/runtime/src/zune.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::{anyhow, Context, Error}; use hotg_rune_compiler::parse::yaml::*; -use hotg_rune_core::{TFLITE_MIMETYPE}; +use hotg_rune_core::TFLITE_MIMETYPE; use hotg_runecoral::{ AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, Tensor as RuneCoralTensor, TensorDescriptor as RuneCoralTensorDescriptor, @@ -18,9 +18,7 @@ use wasmer::{ImportObject, Module, Store}; use zip; pub use self::{proc_block_v1::*, runtime_v1::*}; -use crate::{ - LoadError, -}; +use crate::LoadError; wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); @@ -47,7 +45,7 @@ struct ModelNode { struct ProcBlockNode { node_id: String, context: ProcBlockV1, - shared_state: Arc> + shared_state: Arc>, } pub struct ZuneEngine { @@ -85,7 +83,7 @@ impl ZuneEngine { let runefile = String::from_utf8(read_zip_resource_by_path("Runefile.yml")?) .context("Unable to read Runefile")?; - tracing::debug!(length=runefile.len(), "Read the Rune"); + tracing::debug!(length = runefile.len(), "Read the Rune"); let parsed_runefile = Document::parse(&runefile).context("Unable to parse Runefile")?; @@ -114,18 +112,30 @@ impl ZuneEngine { let graph_contexts = pipeline .iter() .map(|(k, v)| { - - let arguments = - v.args() + let arguments = v + .args() .iter() - .map(|(name, argument)| (name.clone(), argument.to_string())) + .map(|(name, argument)| { + (name.clone(), argument.to_string()) + }) .collect(); - (k.clone(), GraphContext{ arguments, input_tensors: HashMap::new(), output_tensors: HashMap::new() }) + ( + k.clone(), + GraphContext { + arguments, + input_tensors: HashMap::new(), + output_tensors: HashMap::new(), + }, + ) }) .collect(); let tensor_constraints = tensors.iter().map(|_| None).collect(); - let shared_state = Arc::new(Mutex::new(State { tensors, tensor_constraints, graph_contexts })); + let shared_state = Arc::new(Mutex::new(State { + tensors, + tensor_constraints, + graph_contexts, + })); tracing::trace!(?input_tensors, ?output_tensors, "Loaded tensors"); @@ -156,14 +166,15 @@ impl ZuneEngine { #[tracing::instrument(skip_all)] pub fn predict(&mut self) -> Result<(), Error> { for stage_name in &self.processing_order { - let _span = tracing::debug_span!("Running Stage", %stage_name).entered(); + let _span = + tracing::debug_span!("Running Stage", %stage_name).entered(); let stage = self.pipeline.get(stage_name).unwrap(); match stage { Stage::Model(_) => { self.models.get_mut(stage_name).unwrap().run()?; }, - Stage::Capability(_) | Stage::ProcBlock(_) => { + Stage::Capability(_) | Stage::ProcBlock(_) => { self.procblocks.get_mut(stage_name).unwrap().run()?; }, _ => {}, @@ -180,59 +191,98 @@ impl ZuneEngine { return &self.output_nodes; } - pub fn get_input_tensor_names(&self, node_name: &str) -> Result, Error> { + pub fn get_input_tensor_names( + &self, + node_name: &str, + ) -> Result, Error> { let state = self.shared_state.lock().unwrap(); - state.graph_contexts - .get(node_name) - .and_then(|c| { - let tensor_list: Vec = c.input_tensors.iter() - .map(|(k, _)| k.to_string()) - .collect(); + state + .graph_contexts + .get(node_name) + .and_then(|c| { + let tensor_list: Vec = c + .input_tensors + .iter() + .map(|(k, _)| k.to_string()) + .collect(); Some(tensor_list) - }) - .ok_or(anyhow!("Unable to get input tensors")) + }) + .ok_or(anyhow!("Unable to get input tensors")) } - pub fn get_input_tensor(&mut self, node_name: &str, tensor_name: &str) -> Option { + pub fn get_input_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + ) -> Option { let state = self.shared_state.lock().unwrap(); - let tensor_constraint = state.graph_contexts.get(node_name).and_then(|c| c.input_tensors.get(tensor_name)); + let tensor_constraint = state + .graph_contexts + .get(node_name) + .and_then(|c| c.input_tensors.get(tensor_name)); match tensor_constraint { - Some(c) if c.tensor_id.is_some() => state.tensors[c.tensor_id.unwrap()].clone(), - _ => None + Some(c) if c.tensor_id.is_some() => { + state.tensors[c.tensor_id.unwrap()].clone() + }, + _ => None, } } - pub fn set_input_tensor(&mut self, node_name: &str, tensor_name: &str, tensor: &TensorResult) { + pub fn set_input_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + tensor: &TensorResult, + ) { let mut state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(node_name).and_then(|c| c.input_tensors.get(tensor_name).and_then(|c| c.tensor_id.clone())); + let tensor_id = state.graph_contexts.get(node_name).and_then(|c| { + c.input_tensors + .get(tensor_name) + .and_then(|c| c.tensor_id.clone()) + }); match tensor_id { - Some(i) => state.tensors[i] = Some(tensor.clone()), - _ => {} + Some(i) => state.tensors[i] = Some(tensor.clone()), + _ => {}, } } - pub fn get_output_tensor_names(&self, node_name: &str) -> Result, Error> { + pub fn get_output_tensor_names( + &self, + node_name: &str, + ) -> Result, Error> { let state = self.shared_state.lock().unwrap(); - state.graph_contexts - .get(node_name) - .and_then(|c| { - let tensor_list: Vec = c.output_tensors.iter() - .map(|(k, _)| k.to_string()) - .collect(); + state + .graph_contexts + .get(node_name) + .and_then(|c| { + let tensor_list: Vec = c + .output_tensors + .iter() + .map(|(k, _)| k.to_string()) + .collect(); Some(tensor_list) - }) - .ok_or(anyhow!("Unable to get input tensors")) + }) + .ok_or(anyhow!("Unable to get input tensors")) } - pub fn get_output_tensor(&mut self, node_name: &str, tensor_name: &str) -> Option { + pub fn get_output_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + ) -> Option { let state = self.shared_state.lock().unwrap(); - let tensor_constraint = state.graph_contexts.get(node_name).and_then(|c| c.output_tensors.get(tensor_name)); + let tensor_constraint = state + .graph_contexts + .get(node_name) + .and_then(|c| c.output_tensors.get(tensor_name)); match tensor_constraint { - Some(c) if c.tensor_id.is_some() => state.tensors[c.tensor_id.unwrap()].clone(), - _ => None + Some(c) if c.tensor_id.is_some() => { + state.tensors[c.tensor_id.unwrap()].clone() + }, + _ => None, } } @@ -256,19 +306,37 @@ impl ZuneEngine { // .ok() // } - pub fn set_output_tensor(&mut self, node_name: &str, tensor_name: &str, tensor: &TensorResult) { + pub fn set_output_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + tensor: &TensorResult, + ) { let mut state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(node_name).and_then(|c| c.output_tensors.get(tensor_name).and_then(|c| c.tensor_id.clone())); + let tensor_id = state.graph_contexts.get(node_name).and_then(|c| { + c.output_tensors + .get(tensor_name) + .and_then(|c| c.tensor_id.clone()) + }); match tensor_id { - Some(i) => state.tensors[i] = Some(tensor.clone()), - _ => {} + Some(i) => state.tensors[i] = Some(tensor.clone()), + _ => {}, } } } impl ModelNode { - #[tracing::instrument(skip(node_data, model_data, shared_state, input_tensors, output_tensors), level = "debug")] + #[tracing::instrument( + skip( + node_data, + model_data, + shared_state, + input_tensors, + output_tensors + ), + level = "debug" + )] fn load( node_id: &str, node_data: &ModelStage, @@ -304,7 +372,9 @@ impl ModelNode { }; let tensor_constraint_from_descriptor = - |t: &RuneCoralTensorDescriptor, tensor_id: usize| -> TensorConstraint { + |t: &RuneCoralTensorDescriptor, + tensor_id: usize| + -> TensorConstraint { let element_type = get_element_type(t); let dimensions = t.shape.iter().map(|&x| x as usize).collect(); @@ -321,9 +391,13 @@ impl ModelNode { Item = RuneCoralTensorDescriptor, >, pipeline_tensors: &HashMap| - -> Result<(HashSet, HashMap), Error> { + -> Result< + (HashSet, HashMap), + Error, + > { let mut tensor_indices: HashSet = HashSet::new(); - let mut tensor_constraints: HashMap = HashMap::new(); + let mut tensor_constraints: HashMap = + HashMap::new(); let mut i = 0; let mut s = shared_state.lock().unwrap(); @@ -341,10 +415,13 @@ impl ModelNode { let tensor_name = model_tensor.name.to_str().ok(); let tensor_name = match tensor_name { - Some(tensor_name) if tensor_name.len() > 0 => tensor_name.to_string(), - _ => format!("{}", i).to_string() + Some(tensor_name) if tensor_name.len() > 0 => { + tensor_name.to_string() + }, + _ => format!("{}", i).to_string(), }; - let tensor_constraint = tensor_constraint_from_descriptor(&model_tensor, tensor_id); + let tensor_constraint = + tensor_constraint_from_descriptor(&model_tensor, tensor_id); let model_tensor = tensor_from_descriptor(&model_tensor); match s.tensors[tensor_id] { @@ -368,7 +445,7 @@ impl ModelNode { tensor_indices.insert(tensor_id); //FIXME: 2 tensors share same name (/empty name) //then tensor_indices.len() != tensor_constraints.len() - tensor_constraints.insert(tensor_name , tensor_constraint); + tensor_constraints.insert(tensor_name, tensor_constraint); i += 1; } @@ -386,12 +463,20 @@ impl ModelNode { )?; let graph_context = GraphContext { - arguments: node_data.args.iter().map(|(k, v)| (k.clone(), v.to_string())).collect(), + arguments: node_data + .args + .iter() + .map(|(k, v)| (k.clone(), v.to_string())) + .collect(), input_tensors: input_tensor_constraints, - output_tensors: output_tensor_constraints + output_tensors: output_tensor_constraints, }; - shared_state.lock().unwrap().graph_contexts.insert(node_id.to_string(), graph_context); + shared_state + .lock() + .unwrap() + .graph_contexts + .insert(node_id.to_string(), graph_context); Ok(ModelNode { context, @@ -445,7 +530,6 @@ impl ModelNode { } }); - self.context .infer(&inputs, &mut outputs) .map_err(|e| anyhow!(e.to_string())) @@ -459,7 +543,7 @@ impl ProcBlockNode { wasm: &[u8], runtime: &Runtime, input_tensors: &HashMap, - output_tensors: &HashMap + output_tensors: &HashMap, ) -> Result { let shared_state = runtime.shared_state.clone(); let store = Store::default(); @@ -467,7 +551,7 @@ impl ProcBlockNode { add_to_imports(&store, &mut imports, runtime.clone()); let module = - Module::new(&store, wasm).context("Unable to load the module")?; + Module::new(&store, wasm).context("Unable to load the module")?; let (pb, _) = ProcBlockV1::instantiate(&store, &module, &mut imports) .context("Unable to instantiate the WebAssembly module")?; @@ -477,26 +561,33 @@ impl ProcBlockNode { // Assign tensors // TODO: See if this can be more smart. // Not bothering with that for now because tensor names are lost in current Runefile format - shared_state.lock() - .unwrap() - .graph_contexts - .get_mut(node_id) - .and_then(|c| { - c.input_tensors.iter_mut() - .enumerate() - .for_each(|(i, (_, t))| { - input_tensors.get(&key(node_id, Some(i))) - .and_then(|&tensor_index| Some(t.tensor_id = Some(tensor_index))); - }); - - c.output_tensors.iter_mut() - .enumerate() - .for_each(|(i, (_, t))| { - output_tensors.get(&key(node_id, Some(i))) - .and_then(|&tensor_index| Some(t.tensor_id = Some(tensor_index))); - }); - Some(()) - }); + shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(node_id) + .and_then(|c| { + c.input_tensors.iter_mut().enumerate().for_each( + |(i, (_, t))| { + input_tensors.get(&key(node_id, Some(i))).and_then( + |&tensor_index| { + Some(t.tensor_id = Some(tensor_index)) + }, + ); + }, + ); + + c.output_tensors.iter_mut().enumerate().for_each( + |(i, (_, t))| { + output_tensors.get(&key(node_id, Some(i))).and_then( + |&tensor_index| { + Some(t.tensor_id = Some(tensor_index)) + }, + ); + }, + ); + Some(()) + }); Ok(ProcBlockNode { node_id: node_id.to_string(), @@ -514,9 +605,18 @@ impl ProcBlockNode { .map_err(|_| anyhow!("Encountered a Runtime Error"))? .map_err(|e| match e { KernelError::Other(s) => anyhow!(s), - KernelError::InvalidArgument(a) => anyhow!("Invalid argument for {}: {}", &self.node_id, a.name), - KernelError::InvalidInput(i) => anyhow!("Invalid input for {}: {}", &self.node_id, i.name), - KernelError::MissingContext => anyhow!("Unable to retrieve kernel context for {}:", &self.node_id) + KernelError::InvalidArgument(a) => anyhow!( + "Invalid argument for {}: {}", + &self.node_id, + a.name + ), + KernelError::InvalidInput(i) => { + anyhow!("Invalid input for {}: {}", &self.node_id, i.name) + }, + KernelError::MissingContext => anyhow!( + "Unable to retrieve kernel context for {}:", + &self.node_id + ), }) } } @@ -576,7 +676,9 @@ fn instantiate_nodes( let mut models: HashMap = HashMap::new(); let mut procblocks: HashMap = HashMap::new(); - let runtime = Runtime{ shared_state: shared_state.clone() }; + let runtime = Runtime { + shared_state: shared_state.clone(), + }; for item in pipeline { // Collect each output tensor into tensors @@ -584,8 +686,9 @@ fn instantiate_nodes( match item.1 { // Models are handled on the host side, so we treat them separately Stage::Capability(stage) => { - let wasm = read_zip_resource_by_path(&stage.capability.to_string()) - .context("Unable to load the capability")?; + let wasm = + read_zip_resource_by_path(&stage.capability.to_string()) + .context("Unable to load the capability")?; procblocks.insert( stage_name.to_string(), @@ -624,8 +727,9 @@ fn instantiate_nodes( ); }, Stage::ProcBlock(stage) => { - let wasm = read_zip_resource_by_path(&stage.proc_block.to_string()) - .context("Unable to load the proc_block")?; + let wasm = + read_zip_resource_by_path(&stage.proc_block.to_string()) + .context("Unable to load the proc_block")?; procblocks.insert( stage_name.to_string(), @@ -639,7 +743,7 @@ fn instantiate_nodes( ); }, - _ => { } // Do nothing for capabilities/outputs + _ => {}, // Do nothing for capabilities/outputs } } @@ -680,7 +784,6 @@ fn get_tensors( // } // } - // Do a depth first traversal of the tree structure to determine the order // of processing/calling predict() Also allocate the output tensors of // each node along the way @@ -696,7 +799,9 @@ fn get_tensors( } for input in stage.inputs() { - if !nodes_to_visit.contains(&input.name) && !nodes_visited.contains(&input.name) { + if !nodes_to_visit.contains(&input.name) + && !nodes_visited.contains(&input.name) + { nodes_to_visit.push(input.name.clone()); } } @@ -737,7 +842,7 @@ struct Metadata { tags: Vec, arguments: Vec, inputs: Vec, - outputs: Vec + outputs: Vec, } #[derive(Debug, Clone)] @@ -748,28 +853,26 @@ struct ArgumentMetadata { } #[derive(Debug, Clone)] -struct TensorMetadata { - -} +struct TensorMetadata {} #[derive(Debug, Clone)] enum Dimensions { Dynamic, - Fixed(Vec) + Fixed(Vec), } #[derive(Debug, Clone)] struct TensorConstraint { tensor_id: Option, element_type: ElementType, - dimensions: Dimensions + dimensions: Dimensions, } #[derive(Debug, Default, Clone)] struct GraphContext { arguments: HashMap, input_tensors: HashMap, - output_tensors: HashMap + output_tensors: HashMap, } impl runtime_v1::RuntimeV1 for Runtime { @@ -878,9 +981,13 @@ impl runtime_v1::RuntimeV1 for Runtime { todo!() } - fn interpret_as_image(&mut self) -> Self::TensorHint { todo!() } + fn interpret_as_image(&mut self) -> Self::TensorHint { + todo!() + } - fn interpret_as_audio(&mut self) -> Self::TensorHint { todo!() } + fn interpret_as_audio(&mut self) -> Self::TensorHint { + todo!() + } fn supported_shapes( &mut self, @@ -905,7 +1012,9 @@ impl runtime_v1::RuntimeV1 for Runtime { todo!() } - fn non_negative_number(&mut self) -> Self::ArgumentHint { todo!() } + fn non_negative_number(&mut self) -> Self::ArgumentHint { + todo!() + } fn supported_argument_type( &mut self, @@ -914,7 +1023,9 @@ impl runtime_v1::RuntimeV1 for Runtime { todo!() } - fn register_node(&mut self, _metadata: &Self::Metadata) { todo!() } + fn register_node(&mut self, _metadata: &Self::Metadata) { + todo!() + } #[tracing::instrument(skip_all, level = "debug")] fn graph_context_for_node( @@ -941,7 +1052,7 @@ impl runtime_v1::RuntimeV1 for Runtime { .unwrap() .graph_contexts .get(ctx) - .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()) )) + .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()))) } #[tracing::instrument(skip(self, ctx), level = "debug")] @@ -965,10 +1076,16 @@ impl runtime_v1::RuntimeV1 for Runtime { element_type, dimensions: match dimensions { DimensionsParam::Dynamic => Dimensions::Dynamic, - DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) - } - }) - }); + DimensionsParam::Fixed(shape) => Dimensions::Fixed( + shape + .iter() + .map(|&i| i.get() as usize) + .collect(), + ), + }, + }, + ) + }); } #[tracing::instrument(skip(self, ctx), level = "debug")] @@ -989,13 +1106,19 @@ impl runtime_v1::RuntimeV1 for Runtime { name.to_string(), TensorConstraint { tensor_id: None, - element_type: element_type, + element_type, dimensions: match dimensions { DimensionsParam::Dynamic => Dimensions::Dynamic, - DimensionsParam::Fixed(shape) => Dimensions::Fixed(shape.iter().map(|&i| i.get() as usize).collect()) - } - }) - }); + DimensionsParam::Fixed(shape) => Dimensions::Fixed( + shape + .iter() + .map(|&i| i.get() as usize) + .collect(), + ), + }, + }, + ) + }); } #[tracing::instrument(skip_all, level = "debug")] @@ -1003,7 +1126,8 @@ impl runtime_v1::RuntimeV1 for Runtime { &mut self, node_id: &str, ) -> Option { - self.shared_state.lock() + self.shared_state + .lock() .unwrap() .graph_contexts .get(node_id)?; @@ -1021,7 +1145,7 @@ impl runtime_v1::RuntimeV1 for Runtime { .unwrap() .graph_contexts .get(ctx) - .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()) )) + .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()))) } #[tracing::instrument(skip(self, ctx), level = "debug")] @@ -1032,16 +1156,14 @@ impl runtime_v1::RuntimeV1 for Runtime { ) -> Option { let state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(ctx) - .and_then(|c| { - c.input_tensors - .get(name) - .and_then(|v| v.tensor_id ) - }); + let tensor_id = state + .graph_contexts + .get(ctx) + .and_then(|c| c.input_tensors.get(name).and_then(|v| v.tensor_id)); match tensor_id { Some(i) => state.tensors[i].clone(), - _ => None + _ => None, } } @@ -1058,23 +1180,27 @@ impl runtime_v1::RuntimeV1 for Runtime { ) { let mut state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(ctx) - .and_then(|c| { - c.output_tensors - .get(name) - .and_then(|v| v.tensor_id ) - }); + let tensor_id = state + .graph_contexts + .get(ctx) + .and_then(|c| c.output_tensors.get(name).and_then(|v| v.tensor_id)); let dimensions = dimensions.iter().map(|&i| i.get() as u32).collect(); // Todo check tensor constraint if tensor_id.is_some() { - state.tensors[tensor_id.unwrap()] = Some(TensorResult{ element_type, buffer: buffer.to_vec(), dimensions } ); + state.tensors[tensor_id.unwrap()] = Some(TensorResult { + element_type, + buffer: buffer.to_vec(), + dimensions, + }); } } - fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { true } + fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { + true + } fn log( &mut self, From 55acd093bb39ee26899c5233466ca2e1e7c75d16 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 04:38:27 +0800 Subject: [PATCH 69/84] Use WAPM's GraphQL API to find which file to download --- Cargo.lock | 128 ++ crates/compiler/Cargo.toml | 3 +- crates/compiler/src/asset_loader/builtin.rs | 113 +- .../compiler/src/asset_loader/queries.graphql | 9 + .../compiler/src/asset_loader/schema.graphql | 1382 +++++++++++++++++ 5 files changed, 1625 insertions(+), 10 deletions(-) create mode 100644 crates/compiler/src/asset_loader/queries.graphql create mode 100644 crates/compiler/src/asset_loader/schema.graphql diff --git a/Cargo.lock b/Cargo.lock index 4775d3116b..fc913e3c12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,12 @@ version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + [[package]] name = "assert_cmd" version = "2.0.4" @@ -538,6 +544,19 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + [[package]] name = "console" version = "0.15.0" @@ -1009,6 +1028,28 @@ dependencies = [ "serde", ] +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1243,6 +1284,65 @@ dependencies = [ "regex 1.5.6", ] +[[package]] +name = "graphql-introspection-query" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2a4732cf5140bd6c082434494f785a19cfb566ab07d1382c3671f5812fed6d" +dependencies = [ + "serde", +] + +[[package]] +name = "graphql-parser" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5613c31f18676f164112732202124f373bb2103ff017b3b85ca954ea6a66ada" +dependencies = [ + "combine", + "failure", +] + +[[package]] +name = "graphql_client" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b58571cfc3cc42c3e8ff44fc6cfbb6c0dea17ed22d20f9d8f1efc4e8209a3f" +dependencies = [ + "graphql_query_derive", + "reqwest", + "serde", + "serde_json", +] + +[[package]] +name = "graphql_client_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4bf9cd823359d74ad3d3ecf1afd4a975f4ff2f891cdf9a66744606daf52de8c" +dependencies = [ + "graphql-introspection-query", + "graphql-parser", + "heck 0.3.3", + "lazy_static", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "graphql_query_derive" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e56b093bfda71de1da99758b036f4cc811fd2511c8a76f75680e9ffbd2bb4251" +dependencies = [ + "graphql_client_codegen", + "proc-macro2", + "syn", +] + [[package]] name = "h2" version = "0.3.13" @@ -1366,6 +1466,7 @@ dependencies = [ name = "hotg-rune-compiler" version = "0.11.3" dependencies = [ + "graphql_client", "indexmap", "insta", "jsonschema", @@ -3348,6 +3449,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "target-lexicon" version = "0.12.4" @@ -3729,6 +3842,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + [[package]] name = "uriparse" version = "0.6.4" @@ -3802,6 +3924,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index dd5eef85a3..c29a64cbf6 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -9,6 +9,7 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +graphql_client = { version = "0.10.0", features = ["reqwest-blocking"], optional = true } indexmap = { version = "1.8.1", features = ["serde-1"] } once_cell = "1.10.0" queryst = { version = "2.1.0" } @@ -34,4 +35,4 @@ tracing-test = "0.2.1" [features] default = ["builtins"] -builtins = ["reqwest"] +builtins = ["reqwest", "graphql_client"] diff --git a/crates/compiler/src/asset_loader/builtin.rs b/crates/compiler/src/asset_loader/builtin.rs index b99bb6c53a..13ceb11e4f 100644 --- a/crates/compiler/src/asset_loader/builtin.rs +++ b/crates/compiler/src/asset_loader/builtin.rs @@ -1,13 +1,30 @@ use std::path::{Component, Path, PathBuf}; +use graphql_client::{GraphQLQuery, Response}; use reqwest::blocking::Client; use uriparse::{Scheme, URI}; use crate::{ - asset_loader::{AssetLoader, ReadError, WapmUri}, + asset_loader::{ + builtin::lookup_package::{ + LookupPackageGetPackageVersion, + LookupPackageGetPackageVersionModules, + }, + AssetLoader, ReadError, WapmUri, + }, im::Vector, }; +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "src/asset_loader/schema.graphql", + query_path = "src/asset_loader/queries.graphql", + response_derives = "Debug,Serialize" +)] +pub struct LookupPackage; + +const WAPM_REGISTRY: &str = "https://registry.wapm.io/graphql"; + /// An [`AssetLoader`] that loads assets using the basic solution. #[derive(Debug, Clone)] pub struct DefaultAssetLoader { @@ -36,15 +53,16 @@ impl AssetLoader for DefaultAssetLoader { fn read(&self, uri: &URI<'_>) -> Result, ReadError> { match uri.scheme() { Scheme::HTTP | Scheme::HTTPS => { - http_file(&self.client, uri).map_err(ReadError::from) + let url = uri.to_string(); + http_file(&self.client, &url).map_err(ReadError::from) }, Scheme::File => local_file(&self.root_directory, uri.path()), Scheme::Unregistered(u) if u.as_str().is_empty() => { local_file(&self.root_directory, uri.path()) }, Scheme::Unregistered(u) if u.as_str() == "wapm" => { - let _uri: WapmUri = uri.try_into().map_err(ReadError::other)?; - todo!() + let uri = uri.try_into().map_err(ReadError::other)?; + wapm_file(&self.client, uri).map_err(ReadError::other) }, other => Err(ReadError::UnsupportedScheme { scheme: other.as_str().into(), @@ -54,15 +72,92 @@ impl AssetLoader for DefaultAssetLoader { } #[tracing::instrument(skip_all)] -fn http_file( +fn wapm_file( client: &Client, - url: &URI<'_>, -) -> Result, reqwest::Error> { + uri: WapmUri, +) -> Result, WapmDownloadError> { + let WapmUri { + namespace, + package_name, + version, + module, + } = uri; + tracing::debug!(%namespace, %package_name, ?version, "Querying WAPM's GraphQL API"); + + let variables = lookup_package::Variables { + name: format!("{namespace}/{package_name}"), + version, + }; + + let Response { data, errors } = + graphql_client::reqwest::post_graphql_blocking::( + client, + WAPM_REGISTRY, + variables, + )?; + + if let Some(mut errors) = errors { + if !errors.is_empty() { + return Err(WapmDownloadError::GraphQL(errors.remove(0))); + } + } + + let LookupPackageGetPackageVersion { version, modules } = data + .and_then(|r| r.get_package_version) + .ok_or(WapmDownloadError::PackageNotFound)?; + + tracing::debug!(%version, ?modules, "Found a compatible package"); + + let LookupPackageGetPackageVersionModules { public_url, .. } = match module + { + Some(module) => match modules.iter().find(|v| v.name == module) { + Some(v) => v, + None => { + return Err(WapmDownloadError::ModuleNotFound { + requested: module, + names: modules.iter().map(|v| v.name.clone()).collect(), + }) + }, + }, + None => match modules.as_slice() { + [] => return Err(WapmDownloadError::EmptyPackage), + [v] => v, + [..] => { + return Err(WapmDownloadError::MultipleModules { + names: modules.iter().map(|v| v.name.clone()).collect(), + }) + }, + }, + }; + + http_file(client, public_url).map_err(WapmDownloadError::Http) +} + +#[derive(Debug, thiserror::Error)] +enum WapmDownloadError { + #[error("The request failed")] + Http(#[from] reqwest::Error), + #[error("GraphQL")] + GraphQL(graphql_client::Error), + #[error("The package wasn't found")] + PackageNotFound, + #[error("The package contains multiple modules, but the caller didn't say which to use")] + MultipleModules { names: Vec }, + #[error("The package is empty")] + EmptyPackage, + #[error("Couldn't find the \"{requested}\" module in {names:?}")] + ModuleNotFound { + requested: String, + names: Vec, + }, +} + +#[tracing::instrument(skip_all)] +fn http_file(client: &Client, url: &str) -> Result, reqwest::Error> { tracing::info!(%url, "Downloading"); - let response = client.get(url.to_string()).send()?.error_for_status()?; + let response = client.get(url).send()?.error_for_status()?; let status_code = response.status(); - let body = response.bytes()?; let body = Vector::from(&*body); diff --git a/crates/compiler/src/asset_loader/queries.graphql b/crates/compiler/src/asset_loader/queries.graphql new file mode 100644 index 0000000000..5e480cf17a --- /dev/null +++ b/crates/compiler/src/asset_loader/queries.graphql @@ -0,0 +1,9 @@ +query LookupPackage($name: String!, $version: String = "latest") { + getPackageVersion(name: $name, version: $version) { + version + modules { + name + publicUrl + } + } +} diff --git a/crates/compiler/src/asset_loader/schema.graphql b/crates/compiler/src/asset_loader/schema.graphql new file mode 100644 index 0000000000..38ff010c08 --- /dev/null +++ b/crates/compiler/src/asset_loader/schema.graphql @@ -0,0 +1,1382 @@ +type APIToken { + createdAt: DateTime! + id: ID! + identifier: String + lastUsedAt: DateTime + revokedAt: DateTime + user: User! +} + +type APITokenConnection { + # Contains the nodes in this connection. + edges: [APITokenEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `APIToken` and its cursor. +type APITokenEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: APIToken +} + +input AcceptNamespaceCollaboratorInviteInput { + clientMutationId: String + inviteId: ID! +} + +type AcceptNamespaceCollaboratorInvitePayload { + clientMutationId: String + namespaceCollaboratorInvite: NamespaceCollaboratorInvite! +} + +input AcceptPackageCollaboratorInviteInput { + clientMutationId: String + inviteId: ID! +} + +type AcceptPackageCollaboratorInvitePayload { + clientMutationId: String + packageCollaboratorInvite: PackageCollaboratorInvite! +} + +input AcceptPackageTransferRequestInput { + clientMutationId: String + packageTransferRequestId: ID! +} + +type AcceptPackageTransferRequestPayload { + clientMutationId: String + package: Package! + packageTransferRequest: PackageTransferRequest! +} + +type ActivityEvent implements Node { + actorIcon: String! + body: ActivityEventBody! + createdAt: DateTime! + + # The ID of the object + id: ID! +} + +type ActivityEventBody { + ranges: [NodeBodyRange!]! + text: String! +} + +type ActivityEventConnection { + # Contains the nodes in this connection. + edges: [ActivityEventEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `ActivityEvent` and its cursor. +type ActivityEventEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: ActivityEvent +} + +input ArchivePackageInput { + clientMutationId: String + packageId: ID! +} + +type ArchivePackagePayload { + clientMutationId: String + package: Package! +} + +input ChangePackageVersionArchivedStatusInput { + clientMutationId: String + isArchived: Boolean + packageVersionId: ID! +} + +type ChangePackageVersionArchivedStatusPayload { + clientMutationId: String + packageVersion: PackageVersion! +} + +input ChangeUserEmailInput { + clientMutationId: String + newEmail: String! +} + +type ChangeUserEmailPayload { + clientMutationId: String + user: User! +} + +input ChangeUserPasswordInput { + clientMutationId: String + password: String! + + # The token associated to change the password. If not existing it will use the request user by default + token: String +} + +type ChangeUserPasswordPayload { + clientMutationId: String + token: String +} + +input ChangeUserUsernameInput { + clientMutationId: String + + # The new user username + username: String! +} + +type ChangeUserUsernamePayload { + clientMutationId: String + token: String + user: User +} + +input CheckUserExistsInput { + clientMutationId: String + + # The user + user: String! +} + +type CheckUserExistsPayload { + clientMutationId: String + exists: Boolean! + + # The user is only returned if the user input was the username + user: User +} + +type Command { + command: String! + module: PackageVersionModule! + packageVersion: PackageVersion! +} + +input CreateNamespaceInput { + # The namespace avatar + avatar: String + clientMutationId: String + + # The namespace description + description: String + + # The namespace display name + displayName: String + name: String! +} + +type CreateNamespacePayload { + clientMutationId: String + namespace: Namespace! + user: User! +} + +# The `DateTime` scalar type represents a DateTime +# value as specified by +# [iso8601](https://en.wikipedia.org/wiki/ISO_8601). +scalar DateTime + +input DeleteNamespaceInput { + clientMutationId: String + namespaceId: ID! +} + +type DeleteNamespacePayload { + clientMutationId: String + success: Boolean! +} + +type ErrorType { + field: String! + messages: [String!]! +} + +input GenerateAPITokenInput { + clientMutationId: String + identifier: String +} + +type GenerateAPITokenPayload { + clientMutationId: String + token: APIToken + tokenRaw: String + user: User +} + +# The `GenericScalar` scalar type represents a generic +# GraphQL scalar value that could be: +# String, Boolean, Int, Float, List or Object. +scalar GenericScalar + +type GetPasswordResetToken { + user: User + valid: Boolean! +} + +union GlobalObject = Namespace | User + +input InputSignature { + data: String! + publicKeyKeyId: String! +} + +type Interface implements Node { + createdAt: DateTime! + description: String! + displayName: String! + homepage: String + icon: String + + # The ID of the object + id: ID! + lastVersion: InterfaceVersion + name: String! + updatedAt: DateTime! + versions(after: String = null, before: String = null, first: Int = null, last: Int = null, offset: Int = null): InterfaceVersionConnection! +} + +type InterfaceVersion implements Node { + content: String! + createdAt: DateTime! + + # The ID of the object + id: ID! + interface: Interface! + packageVersions(after: String = null, before: String = null, first: Int = null, last: Int = null, offset: Int = null): PackageVersionConnection! + publishedBy: User! + updatedAt: DateTime! + version: String! +} + +type InterfaceVersionConnection { + # Contains the nodes in this connection. + edges: [InterfaceVersionEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `InterfaceVersion` and its cursor. +type InterfaceVersionEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: InterfaceVersion +} + +input InviteNamespaceCollaboratorInput { + clientMutationId: String + email: String + namespaceId: ID! + role: Role! + username: String +} + +type InviteNamespaceCollaboratorPayload { + clientMutationId: String + invite: NamespaceCollaboratorInvite! + namespace: Namespace! +} + +input InvitePackageCollaboratorInput { + clientMutationId: String + email: String + packageName: String! + role: Role! + username: String +} + +type InvitePackageCollaboratorPayload { + clientMutationId: String + invite: PackageCollaboratorInvite! + package: Package! +} + +input LikePackageInput { + clientMutationId: String + packageId: ID! +} + +type LikePackagePayload { + clientMutationId: String + package: Package! +} + +interface Likeable { + id: ID! + likersCount: Int! + viewerHasLiked: Boolean! +} + +type Mutation { + acceptNamespaceCollaboratorInvite(input: AcceptNamespaceCollaboratorInviteInput!): AcceptNamespaceCollaboratorInvitePayload + acceptPackageCollaboratorInvite(input: AcceptPackageCollaboratorInviteInput!): AcceptPackageCollaboratorInvitePayload + acceptPackageTransferRequest(input: AcceptPackageTransferRequestInput!): AcceptPackageTransferRequestPayload + archivePackage(input: ArchivePackageInput!): ArchivePackagePayload + changePackageVersionArchivedStatus(input: ChangePackageVersionArchivedStatusInput!): ChangePackageVersionArchivedStatusPayload + changeUserEmail(input: ChangeUserEmailInput!): ChangeUserEmailPayload + changeUserPassword(input: ChangeUserPasswordInput!): ChangeUserPasswordPayload + changeUserUsername(input: ChangeUserUsernameInput!): ChangeUserUsernamePayload + checkUserExists(input: CheckUserExistsInput!): CheckUserExistsPayload + createNamespace(input: CreateNamespaceInput!): CreateNamespacePayload + deleteNamespace(input: DeleteNamespaceInput!): DeleteNamespacePayload + generateApiToken(input: GenerateAPITokenInput!): GenerateAPITokenPayload + inviteNamespaceCollaborator(input: InviteNamespaceCollaboratorInput!): InviteNamespaceCollaboratorPayload + invitePackageCollaborator(input: InvitePackageCollaboratorInput!): InvitePackageCollaboratorPayload + likePackage(input: LikePackageInput!): LikePackagePayload + publishPackage(input: PublishPackageInput!): PublishPackagePayload + publishPublicKey(input: PublishPublicKeyInput!): PublishPublicKeyPayload + readNotification(input: ReadNotificationInput!): ReadNotificationPayload + refreshToken(input: RefreshInput!): RefreshPayload + registerUser(input: RegisterUserInput!): RegisterUserPayload + removeNamespaceCollaborator(input: RemoveNamespaceCollaboratorInput!): RemoveNamespaceCollaboratorPayload + removeNamespaceCollaboratorInvite(input: RemoveNamespaceCollaboratorInviteInput!): RemoveNamespaceCollaboratorInvitePayload + removePackageCollaborator(input: RemovePackageCollaboratorInput!): RemovePackageCollaboratorPayload + removePackageCollaboratorInvite(input: RemovePackageCollaboratorInviteInput!): RemovePackageCollaboratorInvitePayload + removePackageTransferRequest(input: RemovePackageTransferRequestInput!): RemovePackageTransferRequestPayload + requestPackageTransfer(input: RequestPackageTransferInput!): RequestPackageTransferPayload + requestPasswordReset(input: RequestPasswordResetInput!): RequestPasswordResetPayload + requestValidationEmail(input: RequestValidationEmailInput!): RequestValidationEmailPayload + revokeApiToken(input: RevokeAPITokenInput!): RevokeAPITokenPayload + seePendingNotifications(input: SeePendingNotificationsInput!): SeePendingNotificationsPayload + + # Social Auth for JSON Web Token (JWT) + socialAuth(input: SocialAuthJWTInput!): SocialAuthJWTPayload + + # Obtain JSON Web Token mutation + tokenAuth(input: ObtainJSONWebTokenInput!): ObtainJSONWebTokenPayload + unlikePackage(input: UnlikePackageInput!): UnlikePackagePayload + unwatchPackage(input: UnwatchPackageInput!): UnwatchPackagePayload + updateNamespace(input: UpdateNamespaceInput!): UpdateNamespacePayload + updateNamespaceCollaboratorRole(input: UpdateNamespaceCollaboratorRoleInput!): UpdateNamespaceCollaboratorRolePayload + updatePackage(input: UpdatePackageInput!): UpdatePackagePayload + updatePackageCollaboratorRole(input: UpdatePackageCollaboratorRoleInput!): UpdatePackageCollaboratorRolePayload + updateUserInfo(input: UpdateUserInfoInput!): UpdateUserInfoPayload + validateUserEmail(input: ValidateUserEmailInput!): ValidateUserEmailPayload + validateUserPassword(input: ValidateUserPasswordInput!): ValidateUserPasswordPayload + verifyToken(input: VerifyInput!): VerifyPayload + watchPackage(input: WatchPackageInput!): WatchPackagePayload +} + +type Namespace implements Node & PackageOwner { + avatar: String! + avatarUpdatedAt: DateTime + collaborators(after: String = null, before: String = null, first: Int = null, last: Int = null): NamespaceCollaboratorConnection + createdAt: DateTime! + description: String! + displayName: String + globalName: String! + + # The ID of the object + id: ID! + maintainerInvites: [NamespaceCollaboratorInvite!]! + maintainersWithRoles(after: String = null, before: String = null, first: Int = null, last: Int = null, offset: Int = null): NamespaceMaintainerConnection! + name: String! + packageVersions(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageVersionConnection + packages(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageConnection + pendingInvites(after: String = null, before: String = null, first: Int = null, last: Int = null): NamespaceCollaboratorInviteConnection + publicActivity(after: String = null, before: String = null, first: Int = null, last: Int = null): ActivityEventConnection! + updatedAt: DateTime! + userSet(after: String = null, before: String = null, first: Int = null, last: Int = null, offset: Int = null): UserConnection! + viewerHasRole(role: Role!): Boolean! +} + +type NamespaceCollaborator { + createdAt: DateTime! + id: ID! + invite: NamespaceCollaboratorInvite + namespace: Namespace! + role: RegistryNamespaceMaintainerRoleChoices! + updatedAt: DateTime! + user: User! +} + +type NamespaceCollaboratorConnection { + # Contains the nodes in this connection. + edges: [NamespaceCollaboratorEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `NamespaceCollaborator` and its cursor. +type NamespaceCollaboratorEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: NamespaceCollaborator +} + +type NamespaceCollaboratorInvite { + accepted: NamespaceMaintainer + approvedBy: User + closedAt: DateTime + createdAt: DateTime! + declinedBy: User + expiresAt: DateTime! + id: ID! + inviteEmail: String + namespace: Namespace! + requestedBy: User! + role: RegistryNamespaceMaintainerInviteRoleChoices! + user: User +} + +type NamespaceCollaboratorInviteConnection { + # Contains the nodes in this connection. + edges: [NamespaceCollaboratorInviteEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `NamespaceCollaboratorInvite` and its cursor. +type NamespaceCollaboratorInviteEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: NamespaceCollaboratorInvite +} + +type NamespaceConnection { + # Contains the nodes in this connection. + edges: [NamespaceEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `Namespace` and its cursor. +type NamespaceEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: Namespace +} + +type NamespaceMaintainer implements Node { + createdAt: DateTime! + + # The ID of the object + id: ID! + invite: NamespaceCollaboratorInvite + namespace: Namespace! + role: RegistryNamespaceMaintainerRoleChoices! + updatedAt: DateTime! + user: User! +} + +type NamespaceMaintainerConnection { + # Contains the nodes in this connection. + edges: [NamespaceMaintainerEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `NamespaceMaintainer` and its cursor. +type NamespaceMaintainerEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: NamespaceMaintainer +} + +# An object with an ID +interface Node { + # The ID of the object + id: ID! +} + +type NodeBodyRange { + entity: Node! + length: Int! + offset: Int! +} + +input ObtainJSONWebTokenInput { + clientMutationId: String + password: String! + username: String! +} + +# Obtain JSON Web Token mutation +type ObtainJSONWebTokenPayload { + clientMutationId: String + payload: GenericScalar! + refreshExpiresIn: Int! + refreshToken: String! + token: String! +} + +type Package implements Likeable & Node & PackageOwner { + alias: String + + # The app icon. It should be formatted in the same way as Apple icons + appIcon: String! @deprecated(reason: "Please use icon instead") + collaborators(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageCollaboratorConnection + createdAt: DateTime! + curated: Boolean! + displayName: String! + + # The total number of downloads of the package + downloadsCount: Int + globalName: String! + + # The app icon. It should be formatted in the same way as Apple icons + icon: String! + iconUpdatedAt: DateTime + + # The ID of the object + id: ID! + isTransferring: Boolean! + lastVersion: PackageVersion + likeCount: Int! + likersCount: Int! + maintainers: [User]! @deprecated(reason: "Please use collaborators instead") + name: String! + namespace: String + owner: PackageOwner + ownerObjectId: Int! + + # The name of the package without the owner + packageName: String! + pendingInvites(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageCollaboratorInviteConnection + private: Boolean! + + # The public keys for all the published versions + publicKeys: [PublicKey!]! + updatedAt: DateTime! + versions: [PackageVersion] + viewerHasLiked: Boolean! + viewerHasRole(role: Role!): Boolean! + viewerIsWatching: Boolean! + watchCount: Int! +} + +type PackageCollaborator implements Node { + createdAt: DateTime! + + # The ID of the object + id: ID! + invite: PackageCollaboratorInvite + package: Package! + role: RegistryPackageMaintainerRoleChoices! + updatedAt: DateTime! + user: User! +} + +type PackageCollaboratorConnection { + # Contains the nodes in this connection. + edges: [PackageCollaboratorEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `PackageCollaborator` and its cursor. +type PackageCollaboratorEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: PackageCollaborator +} + +type PackageCollaboratorInvite implements Node { + accepted: PackageCollaborator + approvedBy: User + closedAt: DateTime + createdAt: DateTime! + declinedBy: User + expiresAt: DateTime! + + # The ID of the object + id: ID! + inviteEmail: String + package: Package! + requestedBy: User! + role: RegistryPackageMaintainerInviteRoleChoices! + user: User +} + +type PackageCollaboratorInviteConnection { + # Contains the nodes in this connection. + edges: [PackageCollaboratorInviteEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `PackageCollaboratorInvite` and its cursor. +type PackageCollaboratorInviteEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: PackageCollaboratorInvite +} + +type PackageConnection { + # Contains the nodes in this connection. + edges: [PackageEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +type PackageDistribution { + downloadUrl: String! + size: Int! +} + +# A Relay edge containing a `Package` and its cursor. +type PackageEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: Package +} + +interface PackageOwner { + globalName: String! +} + +type PackageTransferRequest implements Node { + approvedBy: User + closedAt: DateTime + createdAt: DateTime! + declinedBy: User + expiresAt: DateTime! + + # The ID of the object + id: ID! + newOwnerObjectId: Int! + package: Package! + previousOwnerObjectId: Int! + requestedBy: User! +} + +type PackageTransferRequestConnection { + # Contains the nodes in this connection. + edges: [PackageTransferRequestEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `PackageTransferRequest` and its cursor. +type PackageTransferRequestEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: PackageTransferRequest +} + +type PackageVersion implements Node { + commands: [Command!]! + createdAt: DateTime! + description: String! + distribution: PackageDistribution! + file: String! + fileSize: Int! + filesystem: [PackageVersionFilesystem]! + homepage: String + + # The ID of the object + id: ID! + isArchived: Boolean! + isLastVersion: Boolean! + isSigned: Boolean! + license: String + licenseFile: String + manifest: String! + moduleInterfaces: [InterfaceVersion!]! + modules: [PackageVersionModule!]! + package: Package! + publishedBy: User! + readme: String + repository: String + signature: Signature + updatedAt: DateTime! + version: String! +} + +type PackageVersionConnection { + # Contains the nodes in this connection. + edges: [PackageVersionEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `PackageVersion` and its cursor. +type PackageVersionEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: PackageVersion +} + +type PackageVersionFilesystem { + host: String! + wasm: String! +} + +type PackageVersionModule { + abi: String + name: String! + publicUrl: String! + source: String! +} + +# The Relay compliant `PageInfo` type, containing data necessary to paginate this connection. +type PageInfo { + # When paginating forwards, the cursor to continue. + endCursor: String + + # When paginating forwards, are there more items? + hasNextPage: Boolean! + + # When paginating backwards, are there more items? + hasPreviousPage: Boolean! + + # When paginating backwards, the cursor to continue. + startCursor: String +} + +type PublicKey implements Node { + # The ID of the object + id: ID! + key: String! + keyId: String! + owner: User! + revoked: Boolean! + revokedAt: DateTime + uploadedAt: DateTime! + verifyingSignature: Signature +} + +input PublishPackageInput { + clientMutationId: String + description: String! + file: String + homepage: String + + # The package icon + icon: String + license: String + licenseFile: String + manifest: String! + name: String! + readme: String + repository: String + signature: InputSignature + version: String! +} + +type PublishPackagePayload { + clientMutationId: String + packageVersion: PackageVersion! + success: Boolean! +} + +input PublishPublicKeyInput { + clientMutationId: String + key: String! + keyId: String! + verifyingSignatureId: String +} + +type PublishPublicKeyPayload { + clientMutationId: String + publicKey: PublicKey! + success: Boolean! +} + +type Query { + getCommand(name: String!): Command + getCommands(names: [String!]!): [Command] + getContract(name: String!): Interface @deprecated(reason: "Please use getInterface instead") + getContractVersion(name: String!, version: String = null): InterfaceVersion @deprecated(reason: "Please use getInterfaceVersion instead") + getContracts(names: [String!]!): [Interface]! @deprecated(reason: "Please use getInterfaces instead") + getGlobalObject(slug: String!): GlobalObject + getInterface(name: String!): Interface + getInterfaceVersion(name: String!, version: String = "latest"): InterfaceVersion + getInterfaces(names: [String!]!): [Interface]! + getNamespace(name: String!): Namespace + getPackage(name: String!): Package + getPackageVersion(name: String!, version: String = "latest"): PackageVersion + getPackageVersions(names: [String!]!): [PackageVersion] + getPackages(names: [String!]!): [Package]! + getPasswordResetToken(token: String!): GetPasswordResetToken + getUser(username: String!): User + node( + # The ID of the object + id: ID! + ): Node + packages(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageConnection + recentPackageVersions(after: String = null, before: String = null, curated: Boolean = null, first: Int = null, last: Int = null, offset: Int = null): PackageVersionConnection + search(after: String = null, before: String = null, curated: Boolean = null, first: Int = null, hasBindings: Boolean = null, isStandalone: Boolean = null, kind: [SearchKind!] = null, last: Int = null, orderBy: SearchOrderBy = null, publishDate: SearchPublishDate = null, query: String!, sort: SearchOrderSort = null, withInterfaces: [String!] = null): SearchConnection! + searchAutocomplete(after: String = null, before: String = null, first: Int = null, kind: [SearchKind!] = null, last: Int = null, query: String!): SearchConnection! + viewer: User +} + +input ReadNotificationInput { + clientMutationId: String + notificationId: ID! +} + +type ReadNotificationPayload { + clientMutationId: String + notification: UserNotification +} + +input RefreshInput { + clientMutationId: String + refreshToken: String +} + +type RefreshPayload { + clientMutationId: String + payload: GenericScalar! + refreshExpiresIn: Int! + refreshToken: String! + token: String! +} + +input RegisterUserInput { + clientMutationId: String + email: String! + fullName: String! + password: String! + username: String! +} + +type RegisterUserPayload { + clientMutationId: String + token: String +} + +# An enumeration. +enum RegistryNamespaceMaintainerInviteRoleChoices { + # Admin + ADMIN + + # Editor + EDITOR + + # Viewer + VIEWER +} + +# An enumeration. +enum RegistryNamespaceMaintainerRoleChoices { + # Admin + ADMIN + + # Editor + EDITOR + + # Viewer + VIEWER +} + +# An enumeration. +enum RegistryPackageMaintainerInviteRoleChoices { + # Admin + ADMIN + + # Editor + EDITOR + + # Viewer + VIEWER +} + +# An enumeration. +enum RegistryPackageMaintainerRoleChoices { + # Admin + ADMIN + + # Editor + EDITOR + + # Viewer + VIEWER +} + +input RemoveNamespaceCollaboratorInput { + clientMutationId: String + namespaceCollaboratorId: ID! +} + +input RemoveNamespaceCollaboratorInviteInput { + clientMutationId: String + inviteId: ID! +} + +type RemoveNamespaceCollaboratorInvitePayload { + clientMutationId: String + namespace: Namespace! +} + +type RemoveNamespaceCollaboratorPayload { + clientMutationId: String + namespace: Namespace! +} + +input RemovePackageCollaboratorInput { + clientMutationId: String + packageCollaboratorId: ID! +} + +input RemovePackageCollaboratorInviteInput { + clientMutationId: String + inviteId: ID! +} + +type RemovePackageCollaboratorInvitePayload { + clientMutationId: String + package: Package! +} + +type RemovePackageCollaboratorPayload { + clientMutationId: String + package: Package! +} + +input RemovePackageTransferRequestInput { + clientMutationId: String + packageTransferRequestId: ID! +} + +type RemovePackageTransferRequestPayload { + clientMutationId: String + package: Package! +} + +input RequestPackageTransferInput { + clientMutationId: String + newOwnerId: ID! + packageId: ID! +} + +type RequestPackageTransferPayload { + clientMutationId: String + package: Package! +} + +input RequestPasswordResetInput { + clientMutationId: String + email: String! +} + +type RequestPasswordResetPayload { + clientMutationId: String + email: String! + errors: [ErrorType] +} + +input RequestValidationEmailInput { + clientMutationId: String + + # The user id + userId: ID +} + +type RequestValidationEmailPayload { + clientMutationId: String + success: Boolean! + user: User +} + +input RevokeAPITokenInput { + clientMutationId: String + + # The API token ID + tokenId: ID! +} + +type RevokeAPITokenPayload { + clientMutationId: String + success: Boolean + token: APIToken +} + +enum Role { + ADMIN + EDITOR + VIEWER +} + +type SearchConnection { + # Contains the nodes in this connection. + edges: [SearchEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `Search` and its cursor. +type SearchEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: SearchResult +} + +enum SearchKind { + NAMESPACE + PACKAGE + USER +} + +enum SearchOrderBy { + ALPHABETICALLY + PUBLISHED_DATE + SIZE + TOTAL_DOWNLOADS +} + +enum SearchOrderSort { + ASC + DESC +} + +enum SearchPublishDate { + LAST_DAY + LAST_MONTH + LAST_WEEK + LAST_YEAR +} + +union SearchResult = Namespace | PackageVersion | User + +input SeePendingNotificationsInput { + clientMutationId: String +} + +type SeePendingNotificationsPayload { + clientMutationId: String + success: Boolean +} + +type Signature { + createdAt: DateTime! + data: String! + id: ID! + publicKey: PublicKey! +} + +input SocialAuthJWTInput { + accessToken: String! + clientMutationId: String + provider: String! +} + +# Social Auth for JSON Web Token (JWT) +type SocialAuthJWTPayload { + clientMutationId: String + social: SocialNode + token: String +} + +scalar SocialCamelJSON + +type SocialNode implements Node { + created: DateTime! + extraData: SocialCamelJSON + + # The ID of the object + id: ID! + modified: DateTime! + provider: String! + uid: String! + user: User! +} + +type Subscription { + packageVersionCreated(ownerId: ID = null, publishedBy: ID = null): PackageVersion! + userNotificationCreated(userId: ID!): UserNotificationCreated! +} + +input UnlikePackageInput { + clientMutationId: String + packageId: ID! +} + +type UnlikePackagePayload { + clientMutationId: String + package: Package! +} + +input UnwatchPackageInput { + clientMutationId: String + packageId: ID! +} + +type UnwatchPackagePayload { + clientMutationId: String + package: Package! +} + +input UpdateNamespaceCollaboratorRoleInput { + clientMutationId: String + namespaceCollaboratorId: ID! + role: Role! +} + +type UpdateNamespaceCollaboratorRolePayload { + clientMutationId: String + collaborator: NamespaceCollaborator! +} + +input UpdateNamespaceInput { + # The namespace avatar + avatar: String + clientMutationId: String + + # The namespace description + description: String + + # The namespace display name + displayName: String + + # The namespace slug name + name: String + namespaceId: ID! +} + +type UpdateNamespacePayload { + clientMutationId: String + namespace: Namespace! +} + +input UpdatePackageCollaboratorRoleInput { + clientMutationId: String + packageCollaboratorId: ID! + role: Role! +} + +type UpdatePackageCollaboratorRolePayload { + clientMutationId: String + collaborator: PackageCollaborator! +} + +input UpdatePackageInput { + clientMutationId: String + + # The package icon + icon: String + packageId: ID! +} + +type UpdatePackagePayload { + clientMutationId: String + package: Package! +} + +input UpdateUserInfoInput { + # The user avatar + avatar: String + + # The user bio + bio: String + clientMutationId: String + + # The user full name + fullName: String + + # The user Github (it can be the url, or the handle with or without the @) + github: String + + # The user location + location: String + + # The user Twitter (it can be the url, or the handle with or without the @) + twitter: String + + # The user id + userId: ID + + # The user website (it must be a valid url) + websiteUrl: String +} + +type UpdateUserInfoPayload { + clientMutationId: String + user: User +} + +type User implements Node & PackageOwner { + apiTokens(after: String = null, before: String = null, first: Int = null, last: Int = null): APITokenConnection + avatar(size: Int = 80): String! + bio: String + dateJoined: DateTime! + email: String! + firstName: String! + fullName: String! + githubUrl: String + globalName: String! + + # The ID of the object + id: ID! + isEmailValidated: Boolean! + isViewer: Boolean! + lastName: String! + location: String + namespaceInvitesIncoming(after: String = null, before: String = null, first: Int = null, last: Int = null): NamespaceCollaboratorInviteConnection + namespaces(after: String = null, before: String = null, first: Int = null, last: Int = null): NamespaceConnection + notifications(after: String = null, before: String = null, first: Int = null, last: Int = null): UserNotificationConnection + packageInvitesIncoming(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageCollaboratorInviteConnection + packageTransfersIncoming(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageTransferRequestConnection + packageVersions(after: String = null, before: String = null, first: Int = null, last: Int = null): PackageVersionConnection + packages(after: String = null, before: String = null, collaborating: Boolean = null, first: Int = null, last: Int = null): PackageConnection + publicActivity(after: String = null, before: String = null, first: Int = null, last: Int = null): ActivityEventConnection! + twitterUrl: String + + # Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only. + username: String! + websiteUrl: String +} + +type UserConnection { + # Contains the nodes in this connection. + edges: [UserEdge]! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +# A Relay edge containing a `User` and its cursor. +type UserEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: User +} + +type UserNotification implements Node { + body: UserNotificationBody! + createdAt: DateTime! + icon: String + + # The ID of the object + id: ID! + kind: UserNotificationKind + seenState: UserNotificationSeenState! +} + +type UserNotificationBody { + ranges: [NodeBodyRange]! + text: String! +} + +type UserNotificationConnection { + # Contains the nodes in this connection. + edges: [UserNotificationEdge]! + hasPendingNotifications: Boolean! + + # Pagination data for this connection. + pageInfo: PageInfo! +} + +type UserNotificationCreated { + notification: UserNotification + notificationDeletedId: ID +} + +# A Relay edge containing a `UserNotification` and its cursor. +type UserNotificationEdge { + # A cursor for use in pagination + cursor: String! + + # The item at the end of the edge + node: UserNotification +} + +union UserNotificationKind = UserNotificationKindIncomingPackageInvite | UserNotificationKindIncomingPackageTransfer | UserNotificationKindPublishedPackageVersion + +type UserNotificationKindIncomingPackageInvite { + packageInvite: PackageCollaboratorInvite! +} + +type UserNotificationKindIncomingPackageTransfer { + packageTransferRequest: PackageTransferRequest! +} + +type UserNotificationKindPublishedPackageVersion { + packageVersion: PackageVersion! +} + +enum UserNotificationSeenState { + SEEN + SEEN_AND_READ + UNSEEN +} + +input ValidateUserEmailInput { + challenge: String! + clientMutationId: String + + # The user id + userId: ID +} + +type ValidateUserEmailPayload { + clientMutationId: String + user: User +} + +input ValidateUserPasswordInput { + clientMutationId: String + password: String! +} + +type ValidateUserPasswordPayload { + clientMutationId: String + success: Boolean +} + +input VerifyInput { + clientMutationId: String + token: String +} + +type VerifyPayload { + clientMutationId: String + payload: GenericScalar! +} + +input WatchPackageInput { + clientMutationId: String + packageId: ID! +} + +type WatchPackagePayload { + clientMutationId: String + package: Package! +} From c4cd24db0b258ae0c59931f3e0ce909eb4424fda Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 03:00:21 +0800 Subject: [PATCH 70/84] Moved ProcBlockNode and ModelNode to their own modules --- crates/runtime/src/zune.rs | 1282 ------------------------- crates/runtime/src/zune/mod.rs | 502 ++++++++++ crates/runtime/src/zune/proc_block.rs | 556 +++++++++++ crates/runtime/src/zune/tflite.rs | 266 +++++ 4 files changed, 1324 insertions(+), 1282 deletions(-) delete mode 100644 crates/runtime/src/zune.rs create mode 100644 crates/runtime/src/zune/mod.rs create mode 100644 crates/runtime/src/zune/proc_block.rs create mode 100644 crates/runtime/src/zune/tflite.rs diff --git a/crates/runtime/src/zune.rs b/crates/runtime/src/zune.rs deleted file mode 100644 index 9141bd32ac..0000000000 --- a/crates/runtime/src/zune.rs +++ /dev/null @@ -1,1282 +0,0 @@ -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - io::{Cursor, Read}, - sync::{Arc, Mutex}, -}; - -use anyhow::{anyhow, Context, Error}; -use hotg_rune_compiler::parse::yaml::*; -use hotg_rune_core::TFLITE_MIMETYPE; -use hotg_runecoral::{ - AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, - Tensor as RuneCoralTensor, TensorDescriptor as RuneCoralTensorDescriptor, - TensorMut as RuneCoralTensorMut, -}; -use indexmap::IndexMap; -use wasmer::{ImportObject, Module, Store}; -use zip; - -pub use self::{proc_block_v1::*, runtime_v1::*}; -use crate::LoadError; - -wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); -wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); - -#[derive(Debug, Default, Clone, wasmer::WasmerEnv)] -struct Runtime { - shared_state: Arc>, -} - -#[derive(Debug, Default)] -struct State { - tensors: Vec>, - tensor_constraints: Vec>, - graph_contexts: HashMap, -} - -struct ModelNode { - context: InferenceContext, - input_tensors: HashSet, - output_tensors: HashSet, - shared_state: Arc>, -} - -struct ProcBlockNode { - node_id: String, - context: ProcBlockV1, - shared_state: Arc>, -} - -pub struct ZuneEngine { - input_nodes: Vec, - output_nodes: Vec, - models: HashMap, - procblocks: HashMap, - pipeline: IndexMap, - processing_order: Vec, - shared_state: Arc>, // resources -} - -impl ZuneEngine { - #[tracing::instrument(skip_all)] - pub fn load(binary: &[u8]) -> Result - where - Self: Sized, - { - let mut archive = zip::ZipArchive::new(Cursor::new(binary)) - .context("Unable to load Zune")?; - - let mut read_zip_resource_by_path = - |path: &str| -> Result, Error> { - let mut requested_file = - archive.by_name(path).with_context(|| { - anyhow!("Unable to find {} in zune", path) - })?; - let mut buffer = Vec::new(); - requested_file.read_to_end(&mut buffer).with_context(|| { - anyhow!("Unable to read {} from zune", path) - })?; - Ok(buffer) - }; - - let runefile = - String::from_utf8(read_zip_resource_by_path("Runefile.yml")?) - .context("Unable to read Runefile")?; - tracing::debug!(length = runefile.len(), "Read the Rune"); - - let parsed_runefile = - Document::parse(&runefile).context("Unable to parse Runefile")?; - let pipeline = &parsed_runefile.to_v1().pipeline; - - let inputs: Vec<_> = pipeline - .iter() - .filter_map(|(k, v)| match v { - Stage::Capability(_) => Some(k.clone()), - _ => None, - }) - .collect(); - - let outputs: Vec<_> = pipeline - .iter() - .filter_map(|(k, v)| match v { - Stage::Out(_) => Some(k.clone()), - _ => None, - }) - .collect(); - - let (tensors, input_tensors, output_tensors, processing_order) = - get_tensors(&inputs, &outputs, &pipeline) - .context(anyhow!("Unable to map out input/output tensors"))?; - - let graph_contexts = pipeline - .iter() - .map(|(k, v)| { - let arguments = v - .args() - .iter() - .map(|(name, argument)| { - (name.clone(), argument.to_string()) - }) - .collect(); - ( - k.clone(), - GraphContext { - arguments, - input_tensors: HashMap::new(), - output_tensors: HashMap::new(), - }, - ) - }) - .collect(); - - let tensor_constraints = tensors.iter().map(|_| None).collect(); - let shared_state = Arc::new(Mutex::new(State { - tensors, - tensor_constraints, - graph_contexts, - })); - - tracing::trace!(?input_tensors, ?output_tensors, "Loaded tensors"); - - let (model_contexts, procblock_contexts) = instantiate_nodes( - pipeline, - read_zip_resource_by_path, - &shared_state, - input_tensors, - output_tensors, - ) - .map_err(LoadError::Other)?; - - tracing::debug!(order=?processing_order, "Determined the execution order"); - - // TODO: Validate and allocate input/output tensors - - Ok(ZuneEngine { - input_nodes: inputs, - output_nodes: outputs, - models: model_contexts, - procblocks: procblock_contexts, - pipeline: pipeline.to_owned(), - processing_order, - shared_state, - }) - } - - #[tracing::instrument(skip_all)] - pub fn predict(&mut self) -> Result<(), Error> { - for stage_name in &self.processing_order { - let _span = - tracing::debug_span!("Running Stage", %stage_name).entered(); - - let stage = self.pipeline.get(stage_name).unwrap(); - match stage { - Stage::Model(_) => { - self.models.get_mut(stage_name).unwrap().run()?; - }, - Stage::Capability(_) | Stage::ProcBlock(_) => { - self.procblocks.get_mut(stage_name).unwrap().run()?; - }, - _ => {}, - } - } - Ok(()) - } - - pub fn input_nodes(&self) -> &'_ Vec { - return &self.input_nodes; - } - - pub fn output_nodes(&self) -> &'_ Vec { - return &self.output_nodes; - } - - pub fn get_input_tensor_names( - &self, - node_name: &str, - ) -> Result, Error> { - let state = self.shared_state.lock().unwrap(); - state - .graph_contexts - .get(node_name) - .and_then(|c| { - let tensor_list: Vec = c - .input_tensors - .iter() - .map(|(k, _)| k.to_string()) - .collect(); - Some(tensor_list) - }) - .ok_or(anyhow!("Unable to get input tensors")) - } - - pub fn get_input_tensor( - &mut self, - node_name: &str, - tensor_name: &str, - ) -> Option { - let state = self.shared_state.lock().unwrap(); - let tensor_constraint = state - .graph_contexts - .get(node_name) - .and_then(|c| c.input_tensors.get(tensor_name)); - - match tensor_constraint { - Some(c) if c.tensor_id.is_some() => { - state.tensors[c.tensor_id.unwrap()].clone() - }, - _ => None, - } - } - - pub fn set_input_tensor( - &mut self, - node_name: &str, - tensor_name: &str, - tensor: &TensorResult, - ) { - let mut state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(node_name).and_then(|c| { - c.input_tensors - .get(tensor_name) - .and_then(|c| c.tensor_id.clone()) - }); - - match tensor_id { - Some(i) => state.tensors[i] = Some(tensor.clone()), - _ => {}, - } - } - - pub fn get_output_tensor_names( - &self, - node_name: &str, - ) -> Result, Error> { - let state = self.shared_state.lock().unwrap(); - state - .graph_contexts - .get(node_name) - .and_then(|c| { - let tensor_list: Vec = c - .output_tensors - .iter() - .map(|(k, _)| k.to_string()) - .collect(); - Some(tensor_list) - }) - .ok_or(anyhow!("Unable to get input tensors")) - } - - pub fn get_output_tensor( - &mut self, - node_name: &str, - tensor_name: &str, - ) -> Option { - let state = self.shared_state.lock().unwrap(); - let tensor_constraint = state - .graph_contexts - .get(node_name) - .and_then(|c| c.output_tensors.get(tensor_name)); - - match tensor_constraint { - Some(c) if c.tensor_id.is_some() => { - state.tensors[c.tensor_id.unwrap()].clone() - }, - _ => None, - } - } - - // pub fn get_tensor(&self, tensor_id: usize) -> Option<&TensorResult> { - // self.shared_state - // .lock() - // .unwrap() - // .tensors - // .get(tensor_id) - // .unwrap_or(&None) - // .as_ref() - // } - - // pub fn set_tensor(&mut self, tensor_id: usize, tensor: &TensorResult) -> Result<(), Error> { - // self.shared_state - // .lock() - // .unwrap() - // .tensors - // .get_mut(tensor_id) - // .and_then(|t| { t = Some(tensor.clone()); Ok() }) - // .ok() - // } - - pub fn set_output_tensor( - &mut self, - node_name: &str, - tensor_name: &str, - tensor: &TensorResult, - ) { - let mut state = self.shared_state.lock().unwrap(); - let tensor_id = state.graph_contexts.get(node_name).and_then(|c| { - c.output_tensors - .get(tensor_name) - .and_then(|c| c.tensor_id.clone()) - }); - - match tensor_id { - Some(i) => state.tensors[i] = Some(tensor.clone()), - _ => {}, - } - } -} - -impl ModelNode { - #[tracing::instrument( - skip( - node_data, - model_data, - shared_state, - input_tensors, - output_tensors - ), - level = "debug" - )] - fn load( - node_id: &str, - node_data: &ModelStage, - model_data: &[u8], - shared_state: &Arc>, - input_tensors: &HashMap, - output_tensors: &HashMap, - ) -> Result { - // Create Inference Context - let context = InferenceContext::create_context( - TFLITE_MIMETYPE, - &model_data, - AccelerationBackend::NONE, - ) - .with_context(|| { - format!( - "Error Instantiating model from zune for stage: {}", - &node_id - ) - })?; - - let tensor_from_descriptor = - |t: &RuneCoralTensorDescriptor| -> TensorResult { - let element_type = get_element_type(t); - let dimensions = t.shape.iter().map(|&x| x as u32).collect(); - let buffer_size = get_buffer_size(element_type, &dimensions); - - TensorResult { - element_type, - dimensions, - buffer: vec![0; buffer_size], - } - }; - - let tensor_constraint_from_descriptor = - |t: &RuneCoralTensorDescriptor, - tensor_id: usize| - -> TensorConstraint { - let element_type = get_element_type(t); - let dimensions = t.shape.iter().map(|&x| x as usize).collect(); - - TensorConstraint { - tensor_id: Some(tensor_id), - element_type, - dimensions: Dimensions::Fixed(dimensions), - } - }; - - // Returns the list of tensor indices in the State's tensors - let allocate_tensors = |tensor_type: &str, - model_tensors: &mut dyn Iterator< - Item = RuneCoralTensorDescriptor, - >, - pipeline_tensors: &HashMap| - -> Result< - (HashSet, HashMap), - Error, - > { - let mut tensor_indices: HashSet = HashSet::new(); - let mut tensor_constraints: HashMap = - HashMap::new(); - let mut i = 0; - let mut s = shared_state.lock().unwrap(); - - while let Some(model_tensor) = model_tensors.next() { - let tensor_key = key(&node_id, Some(i)); - let tensor_id = - *pipeline_tensors.get(&tensor_key).ok_or_else(|| { - anyhow!( - "Unable to find pipeline_tensor for {} tensor \ - with key {}", - &tensor_type, - &tensor_key - ) - })?; - - let tensor_name = model_tensor.name.to_str().ok(); - let tensor_name = match tensor_name { - Some(tensor_name) if tensor_name.len() > 0 => { - tensor_name.to_string() - }, - _ => format!("{}", i).to_string(), - }; - let tensor_constraint = - tensor_constraint_from_descriptor(&model_tensor, tensor_id); - let model_tensor = tensor_from_descriptor(&model_tensor); - - match s.tensors[tensor_id] { - Some(ref t) - if t.dimensions != model_tensor.dimensions - || t.element_type != model_tensor.element_type => - { - return Err(anyhow!( - "Pipeline tensor for {} with key {} doesn't match \ - model tensor", - &tensor_type, - &tensor_key - )) - }, - Some(_) => {}, - ref mut other => { - *other = Some(model_tensor); - }, - } - - tensor_indices.insert(tensor_id); - //FIXME: 2 tensors share same name (/empty name) - //then tensor_indices.len() != tensor_constraints.len() - tensor_constraints.insert(tensor_name, tensor_constraint); - - i += 1; - } - - Ok((tensor_indices, tensor_constraints)) - }; - - let (input_tensors, input_tensor_constraints) = - allocate_tensors("input", &mut context.inputs(), &input_tensors)?; - - let (output_tensors, output_tensor_constraints) = allocate_tensors( - "output", - &mut context.outputs(), - &output_tensors, - )?; - - let graph_context = GraphContext { - arguments: node_data - .args - .iter() - .map(|(k, v)| (k.clone(), v.to_string())) - .collect(), - input_tensors: input_tensor_constraints, - output_tensors: output_tensor_constraints, - }; - - shared_state - .lock() - .unwrap() - .graph_contexts - .insert(node_id.to_string(), graph_context); - - Ok(ModelNode { - context, - input_tensors, - output_tensors, - shared_state: shared_state.clone(), - }) - } - - #[tracing::instrument(skip_all, level = "debug")] - fn run(&mut self) -> Result<(), Error> { - // We are recreating the input_tensors and output_tensors every time - // before predict because wasm linear memory might have changed - // the locations TODO: There's an optimization that can happen - // here.. but just not yet - let mut inputs: Vec = Vec::new(); - let mut outputs: Vec = Vec::new(); - let mut state = self.shared_state.lock().unwrap(); - - state.tensors.iter_mut().enumerate().for_each(|(i, t)| { - if self.input_tensors.contains(&i) { - let pipeline_tensor = t.as_mut().unwrap(); - unsafe { - inputs.push(RuneCoralTensor { - element_type: get_runecoral_element_type( - &pipeline_tensor.element_type, - ), - shape: Cow::Borrowed(std::slice::from_raw_parts( - pipeline_tensor.dimensions.as_ptr() as *const i32, - pipeline_tensor.dimensions.len(), - )), - buffer: &pipeline_tensor.buffer, - }) - } - } else if self.output_tensors.contains(&i) { - let pipeline_tensor = t.as_mut().unwrap(); - unsafe { - outputs.push(RuneCoralTensorMut { - element_type: get_runecoral_element_type( - &pipeline_tensor.element_type, - ), - shape: Cow::Borrowed(std::slice::from_raw_parts( - pipeline_tensor.dimensions.as_ptr() as *const i32, - pipeline_tensor.dimensions.len(), - )), - buffer: &mut pipeline_tensor.buffer, - }) - } - } else { - // Do nothing - } - }); - - self.context - .infer(&inputs, &mut outputs) - .map_err(|e| anyhow!(e.to_string())) - } -} - -impl ProcBlockNode { - #[tracing::instrument(skip_all, level = "debug", fields(%node_id))] - fn load( - node_id: &str, - wasm: &[u8], - runtime: &Runtime, - input_tensors: &HashMap, - output_tensors: &HashMap, - ) -> Result { - let shared_state = runtime.shared_state.clone(); - let store = Store::default(); - let mut imports = ImportObject::default(); - add_to_imports(&store, &mut imports, runtime.clone()); - - let module = - Module::new(&store, wasm).context("Unable to load the module")?; - let (pb, _) = - ProcBlockV1::instantiate(&store, &module, &mut imports) - .context("Unable to instantiate the WebAssembly module")?; - - let _result = pb.graph(node_id); - - // Assign tensors - // TODO: See if this can be more smart. - // Not bothering with that for now because tensor names are lost in current Runefile format - shared_state - .lock() - .unwrap() - .graph_contexts - .get_mut(node_id) - .and_then(|c| { - c.input_tensors.iter_mut().enumerate().for_each( - |(i, (_, t))| { - input_tensors.get(&key(node_id, Some(i))).and_then( - |&tensor_index| { - Some(t.tensor_id = Some(tensor_index)) - }, - ); - }, - ); - - c.output_tensors.iter_mut().enumerate().for_each( - |(i, (_, t))| { - output_tensors.get(&key(node_id, Some(i))).and_then( - |&tensor_index| { - Some(t.tensor_id = Some(tensor_index)) - }, - ); - }, - ); - Some(()) - }); - - Ok(ProcBlockNode { - node_id: node_id.to_string(), - context: pb, - shared_state: shared_state.clone(), - }) - } - - #[tracing::instrument(skip_all, level = "debug")] - fn run(&mut self) -> Result<(), Error> { - println!("Executing proc block: {:?} ", self.node_id); - // impl stderr for KernelError - self.context - .kernel(&self.node_id) - .map_err(|_| anyhow!("Encountered a Runtime Error"))? - .map_err(|e| match e { - KernelError::Other(s) => anyhow!(s), - KernelError::InvalidArgument(a) => anyhow!( - "Invalid argument for {}: {}", - &self.node_id, - a.name - ), - KernelError::InvalidInput(i) => { - anyhow!("Invalid input for {}: {}", &self.node_id, i.name) - }, - KernelError::MissingContext => anyhow!( - "Unable to retrieve kernel context for {}:", - &self.node_id - ), - }) - } -} - -fn get_buffer_size(element_type: ElementType, dimensions: &Vec) -> usize { - (dimensions.iter().fold(1, |a, &b| a * b) - * get_bytes_per_element(element_type)) as usize -} - -fn get_bytes_per_element(element_type: ElementType) -> u32 { - match element_type { - ElementType::I16 => 2, - ElementType::I32 | ElementType::F32 => 4, - ElementType::I64 | ElementType::F64 => 8, - _ => 1, - } -} - -fn get_element_type(t: &RuneCoralTensorDescriptor) -> ElementType { - match t.element_type { - RuneCoralElementType::UInt8 => ElementType::U8, - RuneCoralElementType::Int8 => ElementType::I8, - RuneCoralElementType::Int16 => ElementType::I16, - RuneCoralElementType::Int32 => ElementType::I32, - RuneCoralElementType::Float32 => ElementType::F32, - RuneCoralElementType::Int64 => ElementType::I64, - RuneCoralElementType::Float64 => ElementType::F64, - RuneCoralElementType::String => ElementType::Utf8, - // TODO: Implement support for all the element types - _ => ElementType::U8, - } -} - -fn get_runecoral_element_type(t: &ElementType) -> RuneCoralElementType { - match t { - ElementType::U8 => RuneCoralElementType::UInt8, - ElementType::I8 => RuneCoralElementType::Int8, - ElementType::I16 => RuneCoralElementType::Int16, - ElementType::I32 => RuneCoralElementType::Int32, - ElementType::F32 => RuneCoralElementType::Float32, - ElementType::I64 => RuneCoralElementType::Int64, - ElementType::F64 => RuneCoralElementType::Float64, - ElementType::Utf8 => RuneCoralElementType::String, - // TODO: Implement support for all the element types - _ => RuneCoralElementType::NoType, - } -} - -fn instantiate_nodes( - pipeline: &IndexMap, - mut read_zip_resource_by_path: impl FnMut(&str) -> Result, Error>, - shared_state: &Arc>, - input_tensors: HashMap, - output_tensors: HashMap, -) -> Result<(HashMap, HashMap), Error> -{ - let mut models: HashMap = HashMap::new(); - let mut procblocks: HashMap = HashMap::new(); - - let runtime = Runtime { - shared_state: shared_state.clone(), - }; - - for item in pipeline { - // Collect each output tensor into tensors - let stage_name = item.0; - match item.1 { - // Models are handled on the host side, so we treat them separately - Stage::Capability(stage) => { - let wasm = - read_zip_resource_by_path(&stage.capability.to_string()) - .context("Unable to load the capability")?; - - procblocks.insert( - stage_name.to_string(), - ProcBlockNode::load( - &stage_name, - &wasm, - &runtime, - &input_tensors, - &output_tensors, - )?, - ); - }, - Stage::Model(stage) => { - // Instantiating the model's inference context here because that - // way model_data gets deallocated once we are done with it - // This way memory usage is under control - let model_data = - read_zip_resource_by_path(&stage.model.to_string()) - .with_context(|| { - anyhow!( - "Unable to read model from zune {}", - stage.model - ) - })?; - - models.insert( - stage_name.to_string(), - ModelNode::load( - &stage_name, - &stage, - &model_data, - &shared_state, - &input_tensors, - &output_tensors, - )?, - ); - }, - Stage::ProcBlock(stage) => { - let wasm = - read_zip_resource_by_path(&stage.proc_block.to_string()) - .context("Unable to load the proc_block")?; - - procblocks.insert( - stage_name.to_string(), - ProcBlockNode::load( - &stage_name, - &wasm, - &runtime, - &input_tensors, - &output_tensors, - )?, - ); - }, - - _ => {}, // Do nothing for capabilities/outputs - } - } - - Ok((models, procblocks)) -} - -fn get_tensors( - inputs: &Vec, - outputs: &Vec, - pipeline: &IndexMap, -) -> Result< - ( - Vec>, - HashMap, - HashMap, - Vec, - ), - Error, -> { - let mut nodes_to_visit = outputs.clone(); - let mut nodes_visited = Vec::new(); - let mut tensors: Vec> = Vec::new(); - let mut output_tensors: HashMap = HashMap::new(); - let mut input_tensors: HashMap = HashMap::new(); - - // For Inputs/Capabilities - We create an input so as to be able to inject inputs - for item in inputs { - tensors.push(None); - input_tensors.insert(key(item, Some(0)), tensors.len() - 1); - output_tensors.insert(key(item, Some(0)), tensors.len() - 1); - } - - // // For Outputs - we allocate all the outputs - // for item in outputs { - // for _ in pipeline.get(item).unwrap().output_types() { - // tensors.push(None); - // output_tensors.insert(key(item, Some(0)), tensors.len() - 1); - // } - // } - - // Do a depth first traversal of the tree structure to determine the order - // of processing/calling predict() Also allocate the output tensors of - // each node along the way - while !nodes_to_visit.is_empty() { - let node = nodes_to_visit.pop().unwrap(); - nodes_visited.push(node.clone()); - - let stage = pipeline.get(&node).unwrap(); - for output_index in 0..stage.output_types().len() { - tensors.push(None); - output_tensors - .insert(key(&node, Some(output_index)), tensors.len() - 1); - } - - for input in stage.inputs() { - if !nodes_to_visit.contains(&input.name) - && !nodes_visited.contains(&input.name) - { - nodes_to_visit.push(input.name.clone()); - } - } - } - - // For each stage in the pipeline, since the inputs have to come from the - // outputs of other stages, simply map to the same tensor - for item in pipeline { - // Collect each output tensor into tensors - let stage_name = item.0; - for i in 0..item.1.inputs().len() { - let input = &item.1.inputs()[i]; - let input_key = key(&input.name, input.index); - let &input_tensor_index = output_tensors.get(&input_key).context( - anyhow!("Invalid input key specified: {}", &input_key), - )?; - input_tensors.insert(key(stage_name, Some(i)), input_tensor_index); - } - } - - nodes_visited.reverse(); - - Ok((tensors, input_tensors, output_tensors, nodes_visited)) -} - -fn key(node_name: &str, tensor_index: Option) -> String { - format!("{}.{}", node_name, tensor_index.or(Some(0)).unwrap()) -} - -#[derive(Debug, Clone)] -pub enum Never {} - -#[derive(Debug, Clone)] -struct Metadata { - description: String, - repository: String, - homepage: String, - tags: Vec, - arguments: Vec, - inputs: Vec, - outputs: Vec, -} - -#[derive(Debug, Clone)] -struct ArgumentMetadata { - description: String, - default_value: String, - // hint: Vec -} - -#[derive(Debug, Clone)] -struct TensorMetadata {} - -#[derive(Debug, Clone)] -enum Dimensions { - Dynamic, - Fixed(Vec), -} - -#[derive(Debug, Clone)] -struct TensorConstraint { - tensor_id: Option, - element_type: ElementType, - dimensions: Dimensions, -} - -#[derive(Debug, Default, Clone)] -struct GraphContext { - arguments: HashMap, - input_tensors: HashMap, - output_tensors: HashMap, -} - -impl runtime_v1::RuntimeV1 for Runtime { - type ArgumentHint = Never; - type ArgumentMetadata = Never; - type KernelContext = String; - type Metadata = Metadata; - type Model = Never; - type TensorHint = Never; - type TensorMetadata = TensorMetadata; - type GraphContext = String; - - fn metadata_new(&mut self, _name: &str, _version: &str) -> Self::Metadata { - todo!() - } - - fn metadata_set_description( - &mut self, - _ctx: &Self::Metadata, - _description: &str, - ) { - todo!() - } - - fn metadata_set_repository(&mut self, _ctx: &Self::Metadata, _url: &str) { - todo!() - } - - fn metadata_set_homepage(&mut self, _ctx: &Self::Metadata, _url: &str) { - todo!() - } - - fn metadata_add_tag(&mut self, _ctx: &Self::Metadata, _tag: &str) { - todo!() - } - - fn metadata_add_argument( - &mut self, - _ctx: &Self::Metadata, - _arg: &Self::ArgumentMetadata, - ) { - todo!() - } - - fn metadata_add_input( - &mut self, - _ctx: &Self::Metadata, - _metadata: &Self::TensorMetadata, - ) { - todo!() - } - - fn metadata_add_output( - &mut self, - _ctx: &Self::Metadata, - _metadata: &Self::TensorMetadata, - ) { - todo!() - } - - fn argument_metadata_new(&mut self, _name: &str) -> Self::ArgumentMetadata { - todo!() - } - - fn argument_metadata_set_description( - &mut self, - _ctx: &Self::ArgumentMetadata, - _description: &str, - ) { - todo!() - } - - fn argument_metadata_set_default_value( - &mut self, - _ctx: &Self::ArgumentMetadata, - _default_value: &str, - ) { - todo!() - } - - fn argument_metadata_add_hint( - &mut self, - _ctx: &Self::ArgumentMetadata, - _hint: &Self::ArgumentHint, - ) { - todo!() - } - - fn tensor_metadata_new(&mut self, _name: &str) -> Self::TensorMetadata { - todo!() - } - - fn tensor_metadata_set_description( - &mut self, - _ctx: &Self::TensorMetadata, - _description: &str, - ) { - todo!() - } - - fn tensor_metadata_add_hint( - &mut self, - _ctx: &Self::TensorMetadata, - _hint: &Self::TensorHint, - ) { - todo!() - } - - fn interpret_as_image(&mut self) -> Self::TensorHint { - todo!() - } - - fn interpret_as_audio(&mut self) -> Self::TensorHint { - todo!() - } - - fn supported_shapes( - &mut self, - _supported_element_types: Vec, - _dimensions: DimensionsParam<'_>, - ) -> Self::TensorHint { - todo!() - } - - fn interpret_as_number_in_range( - &mut self, - _min: &str, - _max: &str, - ) -> Self::ArgumentHint { - todo!() - } - - fn interpret_as_string_in_enum( - &mut self, - _string_enum: Vec<&str>, - ) -> Self::ArgumentHint { - todo!() - } - - fn non_negative_number(&mut self) -> Self::ArgumentHint { - todo!() - } - - fn supported_argument_type( - &mut self, - _hint: ArgumentType, - ) -> Self::ArgumentHint { - todo!() - } - - fn register_node(&mut self, _metadata: &Self::Metadata) { - todo!() - } - - #[tracing::instrument(skip_all, level = "debug")] - fn graph_context_for_node( - &mut self, - node_id: &str, - ) -> Option { - self.shared_state - .lock() - .unwrap() - .graph_contexts - .get(node_id)?; - - Some(node_id.to_string()) - } - - #[tracing::instrument(skip(self, ctx), level = "debug")] - fn graph_context_get_argument( - &mut self, - ctx: &Self::GraphContext, - name: &str, - ) -> Option { - self.shared_state - .lock() - .unwrap() - .graph_contexts - .get(ctx) - .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()))) - } - - #[tracing::instrument(skip(self, ctx), level = "debug")] - fn graph_context_add_input_tensor( - &mut self, - ctx: &Self::GraphContext, - name: &str, - element_type: ElementType, - dimensions: DimensionsParam<'_>, - ) { - self.shared_state - .lock() - .unwrap() - .graph_contexts - .get_mut(ctx) - .and_then(|c| { - c.input_tensors.insert( - name.to_string(), - TensorConstraint { - tensor_id: None, - element_type, - dimensions: match dimensions { - DimensionsParam::Dynamic => Dimensions::Dynamic, - DimensionsParam::Fixed(shape) => Dimensions::Fixed( - shape - .iter() - .map(|&i| i.get() as usize) - .collect(), - ), - }, - }, - ) - }); - } - - #[tracing::instrument(skip(self, ctx), level = "debug")] - fn graph_context_add_output_tensor( - &mut self, - ctx: &Self::GraphContext, - name: &str, - element_type: ElementType, - dimensions: DimensionsParam<'_>, - ) { - self.shared_state - .lock() - .unwrap() - .graph_contexts - .get_mut(ctx) - .and_then(|c| { - c.output_tensors.insert( - name.to_string(), - TensorConstraint { - tensor_id: None, - element_type, - dimensions: match dimensions { - DimensionsParam::Dynamic => Dimensions::Dynamic, - DimensionsParam::Fixed(shape) => Dimensions::Fixed( - shape - .iter() - .map(|&i| i.get() as usize) - .collect(), - ), - }, - }, - ) - }); - } - - #[tracing::instrument(skip_all, level = "debug")] - fn kernel_context_for_node( - &mut self, - node_id: &str, - ) -> Option { - self.shared_state - .lock() - .unwrap() - .graph_contexts - .get(node_id)?; - Some(node_id.to_string()) - } - - #[tracing::instrument(skip(self, ctx), level = "debug")] - fn kernel_context_get_argument( - &mut self, - ctx: &Self::KernelContext, - name: &str, - ) -> Option { - self.shared_state - .lock() - .unwrap() - .graph_contexts - .get(ctx) - .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()))) - } - - #[tracing::instrument(skip(self, ctx), level = "debug")] - fn kernel_context_get_input_tensor( - &mut self, - ctx: &Self::KernelContext, - name: &str, - ) -> Option { - let state = self.shared_state.lock().unwrap(); - - let tensor_id = state - .graph_contexts - .get(ctx) - .and_then(|c| c.input_tensors.get(name).and_then(|v| v.tensor_id)); - - match tensor_id { - Some(i) => state.tensors[i].clone(), - _ => None, - } - } - - #[tracing::instrument(skip(self, ctx, buffer), level = "debug")] - fn kernel_context_set_output_tensor( - &mut self, - ctx: &Self::KernelContext, - name: &str, - TensorParam { - element_type, - buffer, - dimensions, - }: TensorParam<'_>, - ) { - let mut state = self.shared_state.lock().unwrap(); - - let tensor_id = state - .graph_contexts - .get(ctx) - .and_then(|c| c.output_tensors.get(name).and_then(|v| v.tensor_id)); - - let dimensions = dimensions.iter().map(|&i| i.get() as u32).collect(); - - // Todo check tensor constraint - - if tensor_id.is_some() { - state.tensors[tensor_id.unwrap()] = Some(TensorResult { - element_type, - buffer: buffer.to_vec(), - dimensions, - }); - } - } - - fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { - true - } - - fn log( - &mut self, - metadata: LogMetadata, - message: &str, - data: Vec<(&'_ str, LogValue<'_>)>, - ) { - let level = match metadata.level { - LogLevel::Trace => tracing::Level::TRACE, - LogLevel::Debug => tracing::Level::DEBUG, - LogLevel::Info => tracing::Level::INFO, - LogLevel::Warn => tracing::Level::WARN, - LogLevel::Error | LogLevel::Fatal => tracing::Level::ERROR, - }; - - let LogMetadata { - name, - target, - level: _, - file, - line, - module, - } = metadata; - - tracing::event!( - tracing::Level::INFO, - meta.level = %level, - meta.name = %name, - meta.target = target, - meta.file = file, - meta.line = line, - meta.module = module, - ?data, - message, - ); - } - - fn kernel_context_get_global_input( - &mut self, - _ctx: &Self::KernelContext, - _name: &str, - ) -> Option { - todo!() - } - - fn kernel_context_set_global_output( - &mut self, - _ctx: &Self::KernelContext, - _name: &str, - _tensor: TensorParam<'_>, - ) { - todo!() - } - - fn model_load( - &mut self, - _: &str, - _: &[u8], - _: Vec<(&str, &str)>, - ) -> Result { - todo!() - } - - fn model_inputs(&mut self, _: &Self::Model) -> Vec { - todo!() - } - - fn model_outputs(&mut self, _: &Self::Model) -> Vec { - todo!() - } - - fn model_infer( - &mut self, - _: &Self::Model, - _: Vec>, - ) -> Result, ModelInferError> { - todo!() - } -} diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs new file mode 100644 index 0000000000..b0c4402852 --- /dev/null +++ b/crates/runtime/src/zune/mod.rs @@ -0,0 +1,502 @@ +mod proc_block; +#[cfg(feature = "tflite")] +mod tflite; + +use std::{ + collections::HashMap, + io::{Cursor, Read}, + sync::{Arc, Mutex}, +}; + +use anyhow::{anyhow, Context, Error}; +use hotg_rune_compiler::parse::yaml::*; +use indexmap::IndexMap; +use zip; + +pub use self::{proc_block_v1::*, runtime_v1::*}; +use crate::{ + zune::proc_block::{GraphContext, ProcBlockNode, TensorConstraint}, + LoadError, +}; + +wit_bindgen_wasmer::export!("../../../wit-files/rune/runtime-v1.wit"); +wit_bindgen_wasmer::import!("../../../wit-files/rune/proc-block-v1.wit"); + +#[derive(Debug, Default, Clone, wasmer::WasmerEnv)] +struct Runtime { + shared_state: Arc>, +} + +#[derive(Debug, Default)] +pub(crate) struct State { + pub(crate) tensors: Vec>, + pub(crate) tensor_constraints: Vec>, + pub(crate) graph_contexts: HashMap, +} + +pub struct ZuneEngine { + input_nodes: Vec, + output_nodes: Vec, + #[cfg(feature = "tflite")] + models: HashMap, + procblocks: HashMap, + pipeline: IndexMap, + processing_order: Vec, + shared_state: Arc>, // resources +} + +impl ZuneEngine { + #[tracing::instrument(skip_all)] + pub fn load(binary: &[u8]) -> Result + where + Self: Sized, + { + let mut archive = zip::ZipArchive::new(Cursor::new(binary)) + .context("Unable to load Zune")?; + + let mut read_zip_resource_by_path = + |path: &str| -> Result, Error> { + let mut requested_file = + archive.by_name(path).with_context(|| { + anyhow!("Unable to find {} in zune", path) + })?; + let mut buffer = Vec::new(); + requested_file.read_to_end(&mut buffer).with_context(|| { + anyhow!("Unable to read {} from zune", path) + })?; + Ok(buffer) + }; + + let runefile = + String::from_utf8(read_zip_resource_by_path("Runefile.yml")?) + .context("Unable to read Runefile")?; + tracing::debug!(length = runefile.len(), "Read the Rune"); + + let parsed_runefile = + Document::parse(&runefile).context("Unable to parse Runefile")?; + let pipeline = &parsed_runefile.to_v1().pipeline; + + let inputs: Vec<_> = pipeline + .iter() + .filter_map(|(k, v)| match v { + Stage::Capability(_) => Some(k.clone()), + _ => None, + }) + .collect(); + + let outputs: Vec<_> = pipeline + .iter() + .filter_map(|(k, v)| match v { + Stage::Out(_) => Some(k.clone()), + _ => None, + }) + .collect(); + + let (tensors, input_tensors, output_tensors, processing_order) = + get_tensors(&inputs, &outputs, &pipeline) + .context(anyhow!("Unable to map out input/output tensors"))?; + + let graph_contexts = pipeline + .iter() + .map(|(k, v)| { + let arguments = v + .args() + .iter() + .map(|(name, argument)| { + (name.clone(), argument.to_string()) + }) + .collect(); + ( + k.clone(), + GraphContext { + arguments, + input_tensors: HashMap::new(), + output_tensors: HashMap::new(), + }, + ) + }) + .collect(); + + let tensor_constraints = tensors.iter().map(|_| None).collect(); + let shared_state = Arc::new(Mutex::new(State { + tensors, + tensor_constraints, + graph_contexts, + })); + + tracing::trace!(?input_tensors, ?output_tensors, "Loaded tensors"); + + let (model_contexts, procblock_contexts) = instantiate_nodes( + pipeline, + read_zip_resource_by_path, + &shared_state, + input_tensors, + output_tensors, + ) + .map_err(LoadError::Other)?; + + tracing::debug!(order=?processing_order, "Determined the execution order"); + + // TODO: Validate and allocate input/output tensors + + Ok(ZuneEngine { + input_nodes: inputs, + output_nodes: outputs, + models: model_contexts, + procblocks: procblock_contexts, + pipeline: pipeline.to_owned(), + processing_order, + shared_state, + }) + } + + #[tracing::instrument(skip_all)] + pub fn predict(&mut self) -> Result<(), Error> { + for stage_name in &self.processing_order { + let _span = + tracing::debug_span!("Running Stage", %stage_name).entered(); + + let stage = self.pipeline.get(stage_name).unwrap(); + match stage { + #[cfg(feature = "tflite")] + Stage::Model(_) => { + self.models.get_mut(stage_name).unwrap().run()?; + }, + #[cfg(not(feature = "tflite"))] + Stage::Model(_) => { + anyhow::bail!( + "Unable to run \"{}\" because models aren't supported", + stage_name + ); + }, + Stage::Capability(_) | Stage::ProcBlock(_) => { + self.procblocks.get_mut(stage_name).unwrap().run()?; + }, + _ => {}, + } + } + Ok(()) + } + + pub fn input_nodes(&self) -> &'_ Vec { + return &self.input_nodes; + } + + pub fn output_nodes(&self) -> &'_ Vec { + return &self.output_nodes; + } + + pub fn get_input_tensor_names( + &self, + node_name: &str, + ) -> Result, Error> { + let state = self.shared_state.lock().unwrap(); + state + .graph_contexts + .get(node_name) + .and_then(|c| { + let tensor_list: Vec = c + .input_tensors + .iter() + .map(|(k, _)| k.to_string()) + .collect(); + Some(tensor_list) + }) + .ok_or(anyhow!("Unable to get input tensors")) + } + + pub fn get_input_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + ) -> Option { + let state = self.shared_state.lock().unwrap(); + let tensor_constraint = state + .graph_contexts + .get(node_name) + .and_then(|c| c.input_tensors.get(tensor_name)); + + match tensor_constraint { + Some(c) if c.tensor_id.is_some() => { + state.tensors[c.tensor_id.unwrap()].clone() + }, + _ => None, + } + } + + pub fn set_input_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + tensor: &TensorResult, + ) { + let mut state = self.shared_state.lock().unwrap(); + let tensor_id = state.graph_contexts.get(node_name).and_then(|c| { + c.input_tensors + .get(tensor_name) + .and_then(|c| c.tensor_id.clone()) + }); + + match tensor_id { + Some(i) => state.tensors[i] = Some(tensor.clone()), + _ => {}, + } + } + + pub fn get_output_tensor_names( + &self, + node_name: &str, + ) -> Result, Error> { + let state = self.shared_state.lock().unwrap(); + state + .graph_contexts + .get(node_name) + .and_then(|c| { + let tensor_list: Vec = c + .output_tensors + .iter() + .map(|(k, _)| k.to_string()) + .collect(); + Some(tensor_list) + }) + .ok_or(anyhow!("Unable to get input tensors")) + } + + pub fn get_output_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + ) -> Option { + let state = self.shared_state.lock().unwrap(); + let tensor_constraint = state + .graph_contexts + .get(node_name) + .and_then(|c| c.output_tensors.get(tensor_name)); + + match tensor_constraint { + Some(c) if c.tensor_id.is_some() => { + state.tensors[c.tensor_id.unwrap()].clone() + }, + _ => None, + } + } + + // pub fn get_tensor(&self, tensor_id: usize) -> Option<&TensorResult> { + // self.shared_state + // .lock() + // .unwrap() + // .tensors + // .get(tensor_id) + // .unwrap_or(&None) + // .as_ref() + // } + + // pub fn set_tensor(&mut self, tensor_id: usize, tensor: &TensorResult) -> Result<(), Error> { + // self.shared_state + // .lock() + // .unwrap() + // .tensors + // .get_mut(tensor_id) + // .and_then(|t| { t = Some(tensor.clone()); Ok() }) + // .ok() + // } + + pub fn set_output_tensor( + &mut self, + node_name: &str, + tensor_name: &str, + tensor: &TensorResult, + ) { + let mut state = self.shared_state.lock().unwrap(); + let tensor_id = state.graph_contexts.get(node_name).and_then(|c| { + c.output_tensors + .get(tensor_name) + .and_then(|c| c.tensor_id.clone()) + }); + + match tensor_id { + Some(i) => state.tensors[i] = Some(tensor.clone()), + _ => {}, + } + } +} + +fn get_buffer_size(element_type: ElementType, dimensions: &Vec) -> usize { + (dimensions.iter().fold(1, |a, &b| a * b) + * get_bytes_per_element(element_type)) as usize +} + +fn get_bytes_per_element(element_type: ElementType) -> u32 { + match element_type { + ElementType::I16 => 2, + ElementType::I32 | ElementType::F32 => 4, + ElementType::I64 | ElementType::F64 => 8, + _ => 1, + } +} + +fn instantiate_nodes( + pipeline: &IndexMap, + mut read_zip_resource_by_path: impl FnMut(&str) -> Result, Error>, + shared_state: &Arc>, + input_tensors: HashMap, + output_tensors: HashMap, +) -> Result<(HashMap, HashMap), Error> +{ + let mut models: HashMap = HashMap::new(); + let mut procblocks: HashMap = HashMap::new(); + + let runtime = Runtime { + shared_state: shared_state.clone(), + }; + + for item in pipeline { + // Collect each output tensor into tensors + let stage_name = item.0; + match item.1 { + // Models are handled on the host side, so we treat them separately + Stage::Capability(stage) => { + let wasm = + read_zip_resource_by_path(&stage.capability.to_string()) + .context("Unable to load the capability")?; + + procblocks.insert( + stage_name.to_string(), + ProcBlockNode::load( + &stage_name, + &wasm, + &runtime, + &input_tensors, + &output_tensors, + )?, + ); + }, + Stage::Model(stage) => { + // Instantiating the model's inference context here because that + // way model_data gets deallocated once we are done with it + // This way memory usage is under control + let model_data = + read_zip_resource_by_path(&stage.model.to_string()) + .with_context(|| { + anyhow!( + "Unable to read model from zune {}", + stage.model + ) + })?; + + models.insert( + stage_name.to_string(), + ModelNode::load( + &stage_name, + &stage, + &model_data, + &shared_state, + &input_tensors, + &output_tensors, + )?, + ); + }, + Stage::ProcBlock(stage) => { + let wasm = + read_zip_resource_by_path(&stage.proc_block.to_string()) + .context("Unable to load the proc_block")?; + + procblocks.insert( + stage_name.to_string(), + ProcBlockNode::load( + &stage_name, + &wasm, + &runtime, + &input_tensors, + &output_tensors, + )?, + ); + }, + + _ => {}, // Do nothing for capabilities/outputs + } + } + + Ok((models, procblocks)) +} + +fn get_tensors( + inputs: &Vec, + outputs: &Vec, + pipeline: &IndexMap, +) -> Result< + ( + Vec>, + HashMap, + HashMap, + Vec, + ), + Error, +> { + let mut nodes_to_visit = outputs.clone(); + let mut nodes_visited = Vec::new(); + let mut tensors: Vec> = Vec::new(); + let mut output_tensors: HashMap = HashMap::new(); + let mut input_tensors: HashMap = HashMap::new(); + + // For Inputs/Capabilities - We create an input so as to be able to inject inputs + for item in inputs { + tensors.push(None); + input_tensors.insert(key(item, Some(0)), tensors.len() - 1); + output_tensors.insert(key(item, Some(0)), tensors.len() - 1); + } + + // // For Outputs - we allocate all the outputs + // for item in outputs { + // for _ in pipeline.get(item).unwrap().output_types() { + // tensors.push(None); + // output_tensors.insert(key(item, Some(0)), tensors.len() - 1); + // } + // } + + // Do a depth first traversal of the tree structure to determine the order + // of processing/calling predict() Also allocate the output tensors of + // each node along the way + while !nodes_to_visit.is_empty() { + let node = nodes_to_visit.pop().unwrap(); + nodes_visited.push(node.clone()); + + let stage = pipeline.get(&node).unwrap(); + for output_index in 0..stage.output_types().len() { + tensors.push(None); + output_tensors + .insert(key(&node, Some(output_index)), tensors.len() - 1); + } + + for input in stage.inputs() { + if !nodes_to_visit.contains(&input.name) + && !nodes_visited.contains(&input.name) + { + nodes_to_visit.push(input.name.clone()); + } + } + } + + // For each stage in the pipeline, since the inputs have to come from the + // outputs of other stages, simply map to the same tensor + for item in pipeline { + // Collect each output tensor into tensors + let stage_name = item.0; + for i in 0..item.1.inputs().len() { + let input = &item.1.inputs()[i]; + let input_key = key(&input.name, input.index); + let &input_tensor_index = output_tensors.get(&input_key).context( + anyhow!("Invalid input key specified: {}", &input_key), + )?; + input_tensors.insert(key(stage_name, Some(i)), input_tensor_index); + } + } + + nodes_visited.reverse(); + + Ok((tensors, input_tensors, output_tensors, nodes_visited)) +} + +fn key(node_name: &str, tensor_index: Option) -> String { + format!("{}.{}", node_name, tensor_index.or(Some(0)).unwrap()) +} diff --git a/crates/runtime/src/zune/proc_block.rs b/crates/runtime/src/zune/proc_block.rs new file mode 100644 index 0000000000..c6beeac06b --- /dev/null +++ b/crates/runtime/src/zune/proc_block.rs @@ -0,0 +1,556 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use anyhow::{Context, Error}; +use wasmer::{ImportObject, Module, Store}; + +use crate::zune::{ + key, runtime_v1, ArgumentType, DimensionsParam, ElementType, KernelError, + LogLevel, LogMetadata, LogValue, ModelInferError, ModelLoadError, + ProcBlockV1, Runtime, State, TensorParam, TensorResult, +}; + +pub(crate) struct ProcBlockNode { + node_id: String, + context: ProcBlockV1, + shared_state: Arc>, +} + +impl ProcBlockNode { + #[tracing::instrument(skip_all, level = "debug", fields(%node_id))] + pub(crate) fn load( + node_id: &str, + wasm: &[u8], + runtime: &Runtime, + input_tensors: &HashMap, + output_tensors: &HashMap, + ) -> Result { + let shared_state = runtime.shared_state.clone(); + let store = Store::default(); + let mut imports = ImportObject::default(); + super::add_to_imports(&store, &mut imports, runtime.clone()); + + let module = + Module::new(&store, wasm).context("Unable to load the module")?; + let (pb, _) = + ProcBlockV1::instantiate(&store, &module, &mut imports) + .context("Unable to instantiate the WebAssembly module")?; + + let _result = pb.graph(node_id); + + // Assign tensors + // TODO: See if this can be more smart. + // Not bothering with that for now because tensor names are lost in current Runefile format + shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(node_id) + .and_then(|c| { + c.input_tensors.iter_mut().enumerate().for_each( + |(i, (_, t))| { + input_tensors.get(&key(node_id, Some(i))).and_then( + |&tensor_index| { + Some(t.tensor_id = Some(tensor_index)) + }, + ); + }, + ); + + c.output_tensors.iter_mut().enumerate().for_each( + |(i, (_, t))| { + output_tensors.get(&key(node_id, Some(i))).and_then( + |&tensor_index| { + Some(t.tensor_id = Some(tensor_index)) + }, + ); + }, + ); + Some(()) + }); + + Ok(ProcBlockNode { + node_id: node_id.to_string(), + context: pb, + shared_state: shared_state.clone(), + }) + } + + #[tracing::instrument(skip_all, level = "debug")] + pub(crate) fn run(&mut self) -> Result<(), Error> { + println!("Executing proc block: {:?} ", self.node_id); + self.context.kernel(&self.node_id)?.map_err(|e| match e { + KernelError::Other(s) => Error::msg(s), + KernelError::InvalidArgument(a) => { + anyhow::anyhow!( + "Invalid argument for {}: {}", + &self.node_id, + a.name + ) + }, + KernelError::InvalidInput(i) => { + anyhow::anyhow!( + "Invalid input for {}: {}", + &self.node_id, + i.name + ) + }, + KernelError::MissingContext => anyhow::anyhow!( + "Unable to retrieve kernel context for {}:", + &self.node_id + ), + }) + } +} + +#[derive(Debug, Clone)] +pub enum Never {} + +#[derive(Debug, Clone)] +struct Metadata { + description: String, + repository: String, + homepage: String, + tags: Vec, + arguments: Vec, + inputs: Vec, + outputs: Vec, +} + +#[derive(Debug, Clone)] +struct ArgumentMetadata { + description: String, + default_value: String, + // hint: Vec +} + +#[derive(Debug, Clone)] +struct TensorMetadata {} + +#[derive(Debug, Clone)] +pub(crate) enum Dimensions { + Dynamic, + Fixed(Vec), +} + +#[derive(Debug, Clone)] +pub(crate) struct TensorConstraint { + pub tensor_id: Option, + pub element_type: ElementType, + pub dimensions: Dimensions, +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct GraphContext { + pub arguments: HashMap, + pub input_tensors: HashMap, + pub output_tensors: HashMap, +} + +impl runtime_v1::RuntimeV1 for Runtime { + type ArgumentHint = Never; + type ArgumentMetadata = Never; + type KernelContext = String; + type Metadata = Metadata; + type Model = Never; + type TensorHint = Never; + type TensorMetadata = TensorMetadata; + type GraphContext = String; + + fn metadata_new(&mut self, _name: &str, _version: &str) -> Self::Metadata { + todo!() + } + + fn metadata_set_description( + &mut self, + _ctx: &Self::Metadata, + _description: &str, + ) { + todo!() + } + + fn metadata_set_repository(&mut self, _ctx: &Self::Metadata, _url: &str) { + todo!() + } + + fn metadata_set_homepage(&mut self, _ctx: &Self::Metadata, _url: &str) { + todo!() + } + + fn metadata_add_tag(&mut self, _ctx: &Self::Metadata, _tag: &str) { + todo!() + } + + fn metadata_add_argument( + &mut self, + _ctx: &Self::Metadata, + _arg: &Self::ArgumentMetadata, + ) { + todo!() + } + + fn metadata_add_input( + &mut self, + _ctx: &Self::Metadata, + _metadata: &Self::TensorMetadata, + ) { + todo!() + } + + fn metadata_add_output( + &mut self, + _ctx: &Self::Metadata, + _metadata: &Self::TensorMetadata, + ) { + todo!() + } + + fn argument_metadata_new(&mut self, _name: &str) -> Self::ArgumentMetadata { + todo!() + } + + fn argument_metadata_set_description( + &mut self, + _ctx: &Self::ArgumentMetadata, + _description: &str, + ) { + todo!() + } + + fn argument_metadata_set_default_value( + &mut self, + _ctx: &Self::ArgumentMetadata, + _default_value: &str, + ) { + todo!() + } + + fn argument_metadata_add_hint( + &mut self, + _ctx: &Self::ArgumentMetadata, + _hint: &Self::ArgumentHint, + ) { + todo!() + } + + fn tensor_metadata_new(&mut self, _name: &str) -> Self::TensorMetadata { + todo!() + } + + fn tensor_metadata_set_description( + &mut self, + _ctx: &Self::TensorMetadata, + _description: &str, + ) { + todo!() + } + + fn tensor_metadata_add_hint( + &mut self, + _ctx: &Self::TensorMetadata, + _hint: &Self::TensorHint, + ) { + todo!() + } + + fn interpret_as_image(&mut self) -> Self::TensorHint { + todo!() + } + + fn interpret_as_audio(&mut self) -> Self::TensorHint { + todo!() + } + + fn supported_shapes( + &mut self, + _supported_element_types: Vec, + _dimensions: DimensionsParam<'_>, + ) -> Self::TensorHint { + todo!() + } + + fn interpret_as_number_in_range( + &mut self, + _min: &str, + _max: &str, + ) -> Self::ArgumentHint { + todo!() + } + + fn interpret_as_string_in_enum( + &mut self, + _string_enum: Vec<&str>, + ) -> Self::ArgumentHint { + todo!() + } + + fn non_negative_number(&mut self) -> Self::ArgumentHint { + todo!() + } + + fn supported_argument_type( + &mut self, + _hint: ArgumentType, + ) -> Self::ArgumentHint { + todo!() + } + + fn register_node(&mut self, _metadata: &Self::Metadata) { + todo!() + } + + #[tracing::instrument(skip_all, level = "debug")] + fn graph_context_for_node( + &mut self, + node_id: &str, + ) -> Option { + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(node_id)?; + + Some(node_id.to_string()) + } + + #[tracing::instrument(skip(self, ctx), level = "debug")] + fn graph_context_get_argument( + &mut self, + ctx: &Self::GraphContext, + name: &str, + ) -> Option { + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(ctx) + .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()))) + } + + #[tracing::instrument(skip(self, ctx), level = "debug")] + fn graph_context_add_input_tensor( + &mut self, + ctx: &Self::GraphContext, + name: &str, + element_type: ElementType, + dimensions: DimensionsParam<'_>, + ) { + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(ctx) + .and_then(|c| { + c.input_tensors.insert( + name.to_string(), + TensorConstraint { + tensor_id: None, + element_type, + dimensions: match dimensions { + DimensionsParam::Dynamic => Dimensions::Dynamic, + DimensionsParam::Fixed(shape) => Dimensions::Fixed( + shape + .iter() + .map(|&i| i.get() as usize) + .collect(), + ), + }, + }, + ) + }); + } + + #[tracing::instrument(skip(self, ctx), level = "debug")] + fn graph_context_add_output_tensor( + &mut self, + ctx: &Self::GraphContext, + name: &str, + element_type: ElementType, + dimensions: DimensionsParam<'_>, + ) { + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(ctx) + .and_then(|c| { + c.output_tensors.insert( + name.to_string(), + TensorConstraint { + tensor_id: None, + element_type, + dimensions: match dimensions { + DimensionsParam::Dynamic => Dimensions::Dynamic, + DimensionsParam::Fixed(shape) => Dimensions::Fixed( + shape + .iter() + .map(|&i| i.get() as usize) + .collect(), + ), + }, + }, + ) + }); + } + + #[tracing::instrument(skip_all, level = "debug")] + fn kernel_context_for_node( + &mut self, + node_id: &str, + ) -> Option { + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(node_id)?; + Some(node_id.to_string()) + } + + #[tracing::instrument(skip(self, ctx), level = "debug")] + fn kernel_context_get_argument( + &mut self, + ctx: &Self::KernelContext, + name: &str, + ) -> Option { + self.shared_state + .lock() + .unwrap() + .graph_contexts + .get(ctx) + .and_then(|c| c.arguments.get(name).and_then(|v| Some(v.clone()))) + } + + #[tracing::instrument(skip(self, ctx), level = "debug")] + fn kernel_context_get_input_tensor( + &mut self, + ctx: &Self::KernelContext, + name: &str, + ) -> Option { + let state = self.shared_state.lock().unwrap(); + + let tensor_id = state + .graph_contexts + .get(ctx) + .and_then(|c| c.input_tensors.get(name).and_then(|v| v.tensor_id)); + + match tensor_id { + Some(i) => state.tensors[i].clone(), + _ => None, + } + } + + #[tracing::instrument(skip(self, ctx, buffer), level = "debug")] + fn kernel_context_set_output_tensor( + &mut self, + ctx: &Self::KernelContext, + name: &str, + TensorParam { + element_type, + buffer, + dimensions, + }: TensorParam<'_>, + ) { + let mut state = self.shared_state.lock().unwrap(); + + let tensor_id = state + .graph_contexts + .get(ctx) + .and_then(|c| c.output_tensors.get(name).and_then(|v| v.tensor_id)); + + let dimensions = dimensions.iter().map(|&i| i.get() as u32).collect(); + + // Todo check tensor constraint + + if tensor_id.is_some() { + state.tensors[tensor_id.unwrap()] = Some(TensorResult { + element_type, + buffer: buffer.to_vec(), + dimensions, + }); + } + } + + fn is_enabled(&mut self, _metadata: LogMetadata) -> bool { + true + } + + fn log( + &mut self, + metadata: LogMetadata, + message: &str, + data: Vec<(&'_ str, LogValue<'_>)>, + ) { + let level = match metadata.level { + LogLevel::Trace => tracing::Level::TRACE, + LogLevel::Debug => tracing::Level::DEBUG, + LogLevel::Info => tracing::Level::INFO, + LogLevel::Warn => tracing::Level::WARN, + LogLevel::Error | LogLevel::Fatal => tracing::Level::ERROR, + }; + + let LogMetadata { + name, + target, + level: _, + file, + line, + module, + } = metadata; + + tracing::event!( + tracing::Level::INFO, + meta.level = %level, + meta.name = %name, + meta.target = target, + meta.file = file, + meta.line = line, + meta.module = module, + ?data, + message, + ); + } + + fn kernel_context_get_global_input( + &mut self, + _ctx: &Self::KernelContext, + _name: &str, + ) -> Option { + todo!() + } + + fn kernel_context_set_global_output( + &mut self, + _ctx: &Self::KernelContext, + _name: &str, + _tensor: TensorParam<'_>, + ) { + todo!() + } + + fn model_load( + &mut self, + _: &str, + _: &[u8], + _: Vec<(&str, &str)>, + ) -> Result { + todo!() + } + + fn model_inputs(&mut self, _: &Self::Model) -> Vec { + todo!() + } + + fn model_outputs(&mut self, _: &Self::Model) -> Vec { + todo!() + } + + fn model_infer( + &mut self, + _: &Self::Model, + _: Vec>, + ) -> Result, ModelInferError> { + todo!() + } +} diff --git a/crates/runtime/src/zune/tflite.rs b/crates/runtime/src/zune/tflite.rs new file mode 100644 index 0000000000..bb97527a8c --- /dev/null +++ b/crates/runtime/src/zune/tflite.rs @@ -0,0 +1,266 @@ +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, +}; + +use anyhow::{Context, Error}; +use hotg_rune_compiler::parse::ModelStage; +use hotg_rune_core::TFLITE_MIMETYPE; +use hotg_runecoral::{ + AccelerationBackend, ElementType as RuneCoralElementType, InferenceContext, + Tensor as RuneCoralTensor, TensorDescriptor as RuneCoralTensorDescriptor, + TensorMut as RuneCoralTensorMut, +}; + +use crate::zune::{ + get_buffer_size, key, proc_block::Dimensions, ElementType, GraphContext, + State, TensorConstraint, TensorResult, +}; + +pub(crate) struct ModelNode { + context: InferenceContext, + input_tensors: HashSet, + output_tensors: HashSet, + shared_state: Arc>, +} + +impl ModelNode { + #[tracing::instrument( + skip( + node_data, + model_data, + shared_state, + input_tensors, + output_tensors + ), + level = "debug" + )] + pub(crate) fn load( + node_id: &str, + node_data: &ModelStage, + model_data: &[u8], + shared_state: &Arc>, + input_tensors: &HashMap, + output_tensors: &HashMap, + ) -> Result { + // Create Inference Context + let context = InferenceContext::create_context( + TFLITE_MIMETYPE, + &model_data, + AccelerationBackend::NONE, + ) + .with_context(|| { + format!( + "Error Instantiating model from zune for stage: {}", + &node_id + ) + })?; + + let tensor_from_descriptor = + |t: &RuneCoralTensorDescriptor| -> TensorResult { + let element_type = get_element_type(t); + let dimensions = t.shape.iter().map(|&x| x as u32).collect(); + let buffer_size = get_buffer_size(element_type, &dimensions); + + TensorResult { + element_type, + dimensions, + buffer: vec![0; buffer_size], + } + }; + + let tensor_constraint_from_descriptor = + |t: &RuneCoralTensorDescriptor, + tensor_id: usize| + -> TensorConstraint { + let element_type = get_element_type(t); + let dimensions = t.shape.iter().map(|&x| x as usize).collect(); + + TensorConstraint { + tensor_id: Some(tensor_id), + element_type, + dimensions: Dimensions::Fixed(dimensions), + } + }; + + // Returns the list of tensor indices in the State's tensors + let allocate_tensors = |tensor_type: &str, + model_tensors: &mut dyn Iterator< + Item = RuneCoralTensorDescriptor, + >, + pipeline_tensors: &HashMap| + -> Result< + (HashSet, HashMap), + Error, + > { + let mut tensor_indices: HashSet = HashSet::new(); + let mut tensor_constraints: HashMap = + HashMap::new(); + let mut i = 0; + let mut s = shared_state.lock().unwrap(); + + while let Some(model_tensor) = model_tensors.next() { + let tensor_key = key(&node_id, Some(i)); + let tensor_id = + *pipeline_tensors.get(&tensor_key).ok_or_else(|| { + anyhow::anyhow!( + "Unable to find pipeline_tensor for {} tensor \ + with key {}", + &tensor_type, + &tensor_key + ) + })?; + + let tensor_name = model_tensor.name.to_str().ok(); + let tensor_name = match tensor_name { + Some(tensor_name) if tensor_name.len() > 0 => { + tensor_name.to_string() + }, + _ => format!("{}", i).to_string(), + }; + let tensor_constraint = + tensor_constraint_from_descriptor(&model_tensor, tensor_id); + let model_tensor = tensor_from_descriptor(&model_tensor); + + match s.tensors[tensor_id] { + Some(ref t) + if t.dimensions != model_tensor.dimensions + || t.element_type != model_tensor.element_type => + { + anyhow::bail!( + "Pipeline tensor for {} with key {} doesn't match \ + model tensor", + &tensor_type, + &tensor_key + ); + }, + Some(_) => {}, + ref mut other => { + *other = Some(model_tensor); + }, + } + + tensor_indices.insert(tensor_id); + //FIXME: 2 tensors share same name (/empty name) + //then tensor_indices.len() != tensor_constraints.len() + tensor_constraints.insert(tensor_name, tensor_constraint); + + i += 1; + } + + Ok((tensor_indices, tensor_constraints)) + }; + + let (input_tensors, input_tensor_constraints) = + allocate_tensors("input", &mut context.inputs(), &input_tensors)?; + + let (output_tensors, output_tensor_constraints) = allocate_tensors( + "output", + &mut context.outputs(), + &output_tensors, + )?; + + let graph_context = GraphContext { + arguments: node_data + .args + .iter() + .map(|(k, v)| (k.clone(), v.to_string())) + .collect(), + input_tensors: input_tensor_constraints, + output_tensors: output_tensor_constraints, + }; + + shared_state + .lock() + .unwrap() + .graph_contexts + .insert(node_id.to_string(), graph_context); + + Ok(ModelNode { + context, + input_tensors, + output_tensors, + shared_state: shared_state.clone(), + }) + } + + #[tracing::instrument(skip_all, level = "debug")] + pub(crate) fn run(&mut self) -> Result<(), Error> { + // We are recreating the input_tensors and output_tensors every time + // before predict because wasm linear memory might have changed + // the locations TODO: There's an optimization that can happen + // here.. but just not yet + let mut inputs: Vec = Vec::new(); + let mut outputs: Vec = Vec::new(); + let mut state = self.shared_state.lock().unwrap(); + + state.tensors.iter_mut().enumerate().for_each(|(i, t)| { + if self.input_tensors.contains(&i) { + let pipeline_tensor = t.as_mut().unwrap(); + unsafe { + inputs.push(RuneCoralTensor { + element_type: get_runecoral_element_type( + &pipeline_tensor.element_type, + ), + shape: Cow::Borrowed(std::slice::from_raw_parts( + pipeline_tensor.dimensions.as_ptr() as *const i32, + pipeline_tensor.dimensions.len(), + )), + buffer: &pipeline_tensor.buffer, + }) + } + } else if self.output_tensors.contains(&i) { + let pipeline_tensor = t.as_mut().unwrap(); + unsafe { + outputs.push(RuneCoralTensorMut { + element_type: get_runecoral_element_type( + &pipeline_tensor.element_type, + ), + shape: Cow::Borrowed(std::slice::from_raw_parts( + pipeline_tensor.dimensions.as_ptr() as *const i32, + pipeline_tensor.dimensions.len(), + )), + buffer: &mut pipeline_tensor.buffer, + }) + } + } else { + // Do nothing + } + }); + + self.context + .infer(&inputs, &mut outputs) + .map_err(Error::from) + } +} + +fn get_element_type(t: &RuneCoralTensorDescriptor) -> ElementType { + match t.element_type { + RuneCoralElementType::UInt8 => ElementType::U8, + RuneCoralElementType::Int8 => ElementType::I8, + RuneCoralElementType::Int16 => ElementType::I16, + RuneCoralElementType::Int32 => ElementType::I32, + RuneCoralElementType::Float32 => ElementType::F32, + RuneCoralElementType::Int64 => ElementType::I64, + RuneCoralElementType::Float64 => ElementType::F64, + RuneCoralElementType::String => ElementType::Utf8, + // TODO: Implement support for all the element types + _ => ElementType::U8, + } +} + +fn get_runecoral_element_type(t: &ElementType) -> RuneCoralElementType { + match t { + ElementType::U8 => RuneCoralElementType::UInt8, + ElementType::I8 => RuneCoralElementType::Int8, + ElementType::I16 => RuneCoralElementType::Int16, + ElementType::I32 => RuneCoralElementType::Int32, + ElementType::F32 => RuneCoralElementType::Float32, + ElementType::I64 => RuneCoralElementType::Int64, + ElementType::F64 => RuneCoralElementType::Float64, + ElementType::Utf8 => RuneCoralElementType::String, + // TODO: Implement support for all the element types + _ => RuneCoralElementType::NoType, + } +} From cf7deedec1aec0bebb332edfccc3adf193076e4e Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 03:21:25 +0800 Subject: [PATCH 71/84] Put ProcBlockNode and ModelNode behind a trait --- crates/runtime/src/zune/mod.rs | 132 ++++++++++++++------------ crates/runtime/src/zune/proc_block.rs | 10 +- crates/runtime/src/zune/tflite.rs | 6 +- 3 files changed, 79 insertions(+), 69 deletions(-) diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs index b0c4402852..2b0d6ad75a 100644 --- a/crates/runtime/src/zune/mod.rs +++ b/crates/runtime/src/zune/mod.rs @@ -22,8 +22,12 @@ use crate::{ wit_bindgen_wasmer::export!("../../../wit-files/rune/runtime-v1.wit"); wit_bindgen_wasmer::import!("../../../wit-files/rune/proc-block-v1.wit"); +pub(crate) trait Node { + fn run(&mut self) -> Result<(), Error>; +} + #[derive(Debug, Default, Clone, wasmer::WasmerEnv)] -struct Runtime { +pub(crate) struct Runtime { shared_state: Arc>, } @@ -37,10 +41,7 @@ pub(crate) struct State { pub struct ZuneEngine { input_nodes: Vec, output_nodes: Vec, - #[cfg(feature = "tflite")] - models: HashMap, - procblocks: HashMap, - pipeline: IndexMap, + nodes: HashMap>, processing_order: Vec, shared_state: Arc>, // resources } @@ -126,7 +127,7 @@ impl ZuneEngine { tracing::trace!(?input_tensors, ?output_tensors, "Loaded tensors"); - let (model_contexts, procblock_contexts) = instantiate_nodes( + let nodes = instantiate_nodes( pipeline, read_zip_resource_by_path, &shared_state, @@ -142,9 +143,7 @@ impl ZuneEngine { Ok(ZuneEngine { input_nodes: inputs, output_nodes: outputs, - models: model_contexts, - procblocks: procblock_contexts, - pipeline: pipeline.to_owned(), + nodes, processing_order, shared_state, }) @@ -156,24 +155,7 @@ impl ZuneEngine { let _span = tracing::debug_span!("Running Stage", %stage_name).entered(); - let stage = self.pipeline.get(stage_name).unwrap(); - match stage { - #[cfg(feature = "tflite")] - Stage::Model(_) => { - self.models.get_mut(stage_name).unwrap().run()?; - }, - #[cfg(not(feature = "tflite"))] - Stage::Model(_) => { - anyhow::bail!( - "Unable to run \"{}\" because models aren't supported", - stage_name - ); - }, - Stage::Capability(_) | Stage::ProcBlock(_) => { - self.procblocks.get_mut(stage_name).unwrap().run()?; - }, - _ => {}, - } + self.nodes.get_mut(stage_name).unwrap().run()?; } Ok(()) } @@ -341,10 +323,8 @@ fn instantiate_nodes( shared_state: &Arc>, input_tensors: HashMap, output_tensors: HashMap, -) -> Result<(HashMap, HashMap), Error> -{ - let mut models: HashMap = HashMap::new(); - let mut procblocks: HashMap = HashMap::new(); +) -> Result>, Error> { + let mut nodes: HashMap> = HashMap::new(); let runtime = Runtime { shared_state: shared_state.clone(), @@ -360,16 +340,14 @@ fn instantiate_nodes( read_zip_resource_by_path(&stage.capability.to_string()) .context("Unable to load the capability")?; - procblocks.insert( - stage_name.to_string(), - ProcBlockNode::load( - &stage_name, - &wasm, - &runtime, - &input_tensors, - &output_tensors, - )?, - ); + let pb = ProcBlockNode::load( + &stage_name, + &wasm, + &runtime, + &input_tensors, + &output_tensors, + )?; + nodes.insert(stage_name.to_string(), Box::new(pb)); }, Stage::Model(stage) => { // Instantiating the model's inference context here because that @@ -384,40 +362,68 @@ fn instantiate_nodes( ) })?; - models.insert( - stage_name.to_string(), - ModelNode::load( - &stage_name, - &stage, - &model_data, - &shared_state, - &input_tensors, - &output_tensors, - )?, - ); + let model_format = + stage.args.get("model-format").map(|f| f.to_string()); + let node = load_model( + &model_data, + model_format.as_deref(), + stage_name, + stage, + shared_state, + &input_tensors, + &output_tensors, + )?; + nodes.insert(stage_name.to_string(), node); }, Stage::ProcBlock(stage) => { let wasm = read_zip_resource_by_path(&stage.proc_block.to_string()) .context("Unable to load the proc_block")?; - procblocks.insert( - stage_name.to_string(), - ProcBlockNode::load( - &stage_name, - &wasm, - &runtime, - &input_tensors, - &output_tensors, - )?, - ); + let pb = ProcBlockNode::load( + &stage_name, + &wasm, + &runtime, + &input_tensors, + &output_tensors, + )?; + nodes.insert(stage_name.to_string(), Box::new(pb)); }, _ => {}, // Do nothing for capabilities/outputs } } - Ok((models, procblocks)) + Ok(nodes) +} + +fn load_model( + model_data: &[u8], + model_format: Option<&str>, + stage_name: &str, + stage: &ModelStage, + shared_state: &Arc>, + input_tensors: &HashMap, + output_tensors: &HashMap, +) -> Result, Error> { + match model_format { + #[cfg(feature = "tflite")] + Some("tflite") | None => { + let model = tflite::ModelNode::load( + stage_name, + stage, + model_data, + shared_state, + input_tensors, + output_tensors, + )?; + + Ok(Box::new(model)) + }, + #[cfg(not(feature = "tflite"))] + None => anyhow::bail!("Unsupported model format, \"tflite\""), + Some(other) => anyhow::bail!("Unsupported model format, \"{}\"", other), + } } fn get_tensors( diff --git a/crates/runtime/src/zune/proc_block.rs b/crates/runtime/src/zune/proc_block.rs index c6beeac06b..85f97320e6 100644 --- a/crates/runtime/src/zune/proc_block.rs +++ b/crates/runtime/src/zune/proc_block.rs @@ -8,7 +8,7 @@ use wasmer::{ImportObject, Module, Store}; use crate::zune::{ key, runtime_v1, ArgumentType, DimensionsParam, ElementType, KernelError, - LogLevel, LogMetadata, LogValue, ModelInferError, ModelLoadError, + LogLevel, LogMetadata, LogValue, ModelInferError, ModelLoadError, Node, ProcBlockV1, Runtime, State, TensorParam, TensorResult, }; @@ -77,9 +77,11 @@ impl ProcBlockNode { shared_state: shared_state.clone(), }) } +} +impl Node for ProcBlockNode { #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run(&mut self) -> Result<(), Error> { + fn run(&mut self) -> Result<(), Error> { println!("Executing proc block: {:?} ", self.node_id); self.context.kernel(&self.node_id)?.map_err(|e| match e { KernelError::Other(s) => Error::msg(s), @@ -109,7 +111,7 @@ impl ProcBlockNode { pub enum Never {} #[derive(Debug, Clone)] -struct Metadata { +pub(crate) struct Metadata { description: String, repository: String, homepage: String, @@ -127,7 +129,7 @@ struct ArgumentMetadata { } #[derive(Debug, Clone)] -struct TensorMetadata {} +pub(crate) struct TensorMetadata {} #[derive(Debug, Clone)] pub(crate) enum Dimensions { diff --git a/crates/runtime/src/zune/tflite.rs b/crates/runtime/src/zune/tflite.rs index bb97527a8c..7ddb94e0c8 100644 --- a/crates/runtime/src/zune/tflite.rs +++ b/crates/runtime/src/zune/tflite.rs @@ -15,7 +15,7 @@ use hotg_runecoral::{ use crate::zune::{ get_buffer_size, key, proc_block::Dimensions, ElementType, GraphContext, - State, TensorConstraint, TensorResult, + Node, State, TensorConstraint, TensorResult, }; pub(crate) struct ModelNode { @@ -184,9 +184,11 @@ impl ModelNode { shared_state: shared_state.clone(), }) } +} +impl Node for ModelNode { #[tracing::instrument(skip_all, level = "debug")] - pub(crate) fn run(&mut self) -> Result<(), Error> { + fn run(&mut self) -> Result<(), Error> { // We are recreating the input_tensors and output_tensors every time // before predict because wasm linear memory might have changed // the locations TODO: There's an optimization that can happen From 58c9137a223580cbcb04803b2f9fe5f8c3be6410 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 03:41:16 +0800 Subject: [PATCH 72/84] Implemented std::error::Error for KernelError and other minor cleanups --- crates/runtime/src/zune/mod.rs | 119 ++++++++++++++++++++++---- crates/runtime/src/zune/proc_block.rs | 25 +----- 2 files changed, 106 insertions(+), 38 deletions(-) diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs index 2b0d6ad75a..1c68971c02 100644 --- a/crates/runtime/src/zune/mod.rs +++ b/crates/runtime/src/zune/mod.rs @@ -4,6 +4,7 @@ mod tflite; use std::{ collections::HashMap, + fmt::{self, Display, Formatter}, io::{Cursor, Read}, sync::{Arc, Mutex}, }; @@ -63,7 +64,7 @@ impl ZuneEngine { })?; let mut buffer = Vec::new(); requested_file.read_to_end(&mut buffer).with_context(|| { - anyhow!("Unable to read {} from zune", path) + format!("Unable to read {} from zune", path) })?; Ok(buffer) }; @@ -95,7 +96,7 @@ impl ZuneEngine { let (tensors, input_tensors, output_tensors, processing_order) = get_tensors(&inputs, &outputs, &pipeline) - .context(anyhow!("Unable to map out input/output tensors"))?; + .context("Unable to map out input/output tensors")?; let graph_contexts = pipeline .iter() @@ -160,11 +161,11 @@ impl ZuneEngine { Ok(()) } - pub fn input_nodes(&self) -> &'_ Vec { + pub fn input_nodes(&self) -> &[String] { return &self.input_nodes; } - pub fn output_nodes(&self) -> &'_ Vec { + pub fn output_nodes(&self) -> &[String] { return &self.output_nodes; } @@ -184,7 +185,7 @@ impl ZuneEngine { .collect(); Some(tensor_list) }) - .ok_or(anyhow!("Unable to get input tensors")) + .context("Unable to get input tensors") } pub fn get_input_tensor( @@ -241,7 +242,7 @@ impl ZuneEngine { .collect(); Some(tensor_list) }) - .ok_or(anyhow!("Unable to get input tensors")) + .context("Unable to get input tensors") } pub fn get_output_tensor( @@ -356,7 +357,7 @@ fn instantiate_nodes( let model_data = read_zip_resource_by_path(&stage.model.to_string()) .with_context(|| { - anyhow!( + format!( "Unable to read model from zune {}", stage.model ) @@ -366,7 +367,7 @@ fn instantiate_nodes( stage.args.get("model-format").map(|f| f.to_string()); let node = load_model( &model_data, - model_format.as_deref(), + model_format.as_deref().unwrap_or("tflite"), stage_name, stage, shared_state, @@ -399,7 +400,7 @@ fn instantiate_nodes( fn load_model( model_data: &[u8], - model_format: Option<&str>, + model_format: &str, stage_name: &str, stage: &ModelStage, shared_state: &Arc>, @@ -408,7 +409,7 @@ fn load_model( ) -> Result, Error> { match model_format { #[cfg(feature = "tflite")] - Some("tflite") | None => { + "tflite" | hotg_rune_core::TFLITE_MIMETYPE => { let model = tflite::ModelNode::load( stage_name, stage, @@ -420,9 +421,7 @@ fn load_model( Ok(Box::new(model)) }, - #[cfg(not(feature = "tflite"))] - None => anyhow::bail!("Unsupported model format, \"tflite\""), - Some(other) => anyhow::bail!("Unsupported model format, \"{}\"", other), + other => anyhow::bail!("Unsupported model format, \"{}\"", other), } } @@ -491,9 +490,10 @@ fn get_tensors( for i in 0..item.1.inputs().len() { let input = &item.1.inputs()[i]; let input_key = key(&input.name, input.index); - let &input_tensor_index = output_tensors.get(&input_key).context( - anyhow!("Invalid input key specified: {}", &input_key), - )?; + let &input_tensor_index = + output_tensors.get(&input_key).with_context(|| { + format!("Invalid input key specified: {}", &input_key) + })?; input_tensors.insert(key(stage_name, Some(i)), input_tensor_index); } } @@ -506,3 +506,90 @@ fn get_tensors( fn key(node_name: &str, tensor_index: Option) -> String { format!("{}.{}", node_name, tensor_index.or(Some(0)).unwrap()) } + +impl std::error::Error for proc_block_v1::GraphError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + GraphError::InvalidArgument(InvalidArgument { reason, .. }) => { + Some(reason) + }, + _ => None, + } + } +} + +impl Display for proc_block_v1::GraphError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + GraphError::Other(msg) => Display::fmt(msg, f), + GraphError::InvalidArgument(InvalidArgument { name, .. }) => { + write!(f, "The \"{}\" argument is invalid", name) + }, + GraphError::MissingContext => { + write!(f, "Unable to retrieve the graph context") + }, + } + } +} + +impl std::error::Error for proc_block_v1::KernelError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + KernelError::InvalidArgument(InvalidArgument { + reason, .. + }) => Some(reason), + KernelError::InvalidInput(InvalidInput { reason, .. }) => { + Some(reason) + }, + _ => None, + } + } +} + +impl Display for proc_block_v1::KernelError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + KernelError::Other(s) => Display::fmt(s, f), + KernelError::InvalidArgument(InvalidArgument { name, .. }) => { + write!(f, "The \"{}\" argument is invalid", name) + }, + KernelError::InvalidInput(InvalidInput { name, .. }) => { + write!(f, "The \"{}\" input is invalid", name) + }, + KernelError::MissingContext => { + write!(f, "Unable to retrieve the kernel context") + }, + } + } +} + +impl std::error::Error for BadArgumentReason {} + +impl Display for BadArgumentReason { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + BadArgumentReason::Other(msg) => Display::fmt(msg, f), + BadArgumentReason::NotFound => write!(f, "Argument not found"), + BadArgumentReason::InvalidValue(msg) => { + write!(f, "Invalid argument value: {}", msg) + }, + } + } +} + +impl std::error::Error for BadInputReason {} + +impl Display for BadInputReason { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + BadInputReason::Other(msg) => Display::fmt(msg, f), + BadInputReason::NotFound => write!(f, "Input not found"), + BadInputReason::UnsupportedShape => { + write!(f, "Unsupported tensor shape") + }, + BadInputReason::InvalidValue(msg) => { + write!(f, "Invalid argument value: {}", msg) + }, + } + } +} diff --git a/crates/runtime/src/zune/proc_block.rs b/crates/runtime/src/zune/proc_block.rs index 85f97320e6..86c414daaa 100644 --- a/crates/runtime/src/zune/proc_block.rs +++ b/crates/runtime/src/zune/proc_block.rs @@ -82,28 +82,9 @@ impl ProcBlockNode { impl Node for ProcBlockNode { #[tracing::instrument(skip_all, level = "debug")] fn run(&mut self) -> Result<(), Error> { - println!("Executing proc block: {:?} ", self.node_id); - self.context.kernel(&self.node_id)?.map_err(|e| match e { - KernelError::Other(s) => Error::msg(s), - KernelError::InvalidArgument(a) => { - anyhow::anyhow!( - "Invalid argument for {}: {}", - &self.node_id, - a.name - ) - }, - KernelError::InvalidInput(i) => { - anyhow::anyhow!( - "Invalid input for {}: {}", - &self.node_id, - i.name - ) - }, - KernelError::MissingContext => anyhow::anyhow!( - "Unable to retrieve kernel context for {}:", - &self.node_id - ), - }) + self.context.kernel(&self.node_id)??; + + Ok(()) } } From 3ca2b049b4dcffd13a77a2b05529ae3ed49e4dcc Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 03:45:19 +0800 Subject: [PATCH 73/84] Removing some dead code --- crates/runtime/src/zune/proc_block.rs | 8 +++----- crates/runtime/src/zune/tflite.rs | 11 +++-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/runtime/src/zune/proc_block.rs b/crates/runtime/src/zune/proc_block.rs index 86c414daaa..982d6fe029 100644 --- a/crates/runtime/src/zune/proc_block.rs +++ b/crates/runtime/src/zune/proc_block.rs @@ -7,15 +7,14 @@ use anyhow::{Context, Error}; use wasmer::{ImportObject, Module, Store}; use crate::zune::{ - key, runtime_v1, ArgumentType, DimensionsParam, ElementType, KernelError, - LogLevel, LogMetadata, LogValue, ModelInferError, ModelLoadError, Node, - ProcBlockV1, Runtime, State, TensorParam, TensorResult, + key, runtime_v1, ArgumentType, DimensionsParam, ElementType, LogLevel, + LogMetadata, LogValue, ModelInferError, ModelLoadError, Node, ProcBlockV1, + Runtime, TensorParam, TensorResult, }; pub(crate) struct ProcBlockNode { node_id: String, context: ProcBlockV1, - shared_state: Arc>, } impl ProcBlockNode { @@ -74,7 +73,6 @@ impl ProcBlockNode { Ok(ProcBlockNode { node_id: node_id.to_string(), context: pb, - shared_state: shared_state.clone(), }) } } diff --git a/crates/runtime/src/zune/tflite.rs b/crates/runtime/src/zune/tflite.rs index 7ddb94e0c8..b2ffe96ecb 100644 --- a/crates/runtime/src/zune/tflite.rs +++ b/crates/runtime/src/zune/tflite.rs @@ -103,13 +103,8 @@ impl ModelNode { while let Some(model_tensor) = model_tensors.next() { let tensor_key = key(&node_id, Some(i)); let tensor_id = - *pipeline_tensors.get(&tensor_key).ok_or_else(|| { - anyhow::anyhow!( - "Unable to find pipeline_tensor for {} tensor \ - with key {}", - &tensor_type, - &tensor_key - ) + *pipeline_tensors.get(&tensor_key).with_context(|| { + format!( "Unable to find pipeline_tensor for {tensor_type} tensor with key {tensor_key}") })?; let tensor_name = model_tensor.name.to_str().ok(); @@ -117,7 +112,7 @@ impl ModelNode { Some(tensor_name) if tensor_name.len() > 0 => { tensor_name.to_string() }, - _ => format!("{}", i).to_string(), + _ => i.to_string(), }; let tensor_constraint = tensor_constraint_from_descriptor(&model_tensor, tensor_id); From d3a3fcf10a04269e137bafe2f2a6863450bc89b7 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 04:45:18 +0800 Subject: [PATCH 74/84] The "zune" module was never put behind its feature flag --- crates/runtime/src/lib.rs | 1 + crates/runtime/src/models/mod.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 8fe1e6f682..54dbbe7684 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -32,6 +32,7 @@ mod tensor; pub mod builtins; mod outputs; +#[cfg(feature = "zune")] pub mod zune; pub use crate::{ diff --git a/crates/runtime/src/models/mod.rs b/crates/runtime/src/models/mod.rs index 59ec7773ce..f92f7f8ad1 100644 --- a/crates/runtime/src/models/mod.rs +++ b/crates/runtime/src/models/mod.rs @@ -15,7 +15,7 @@ use crate::callbacks::{Model, ModelMetadata}; /// /// Supported formats are: /// - TensorFlow Lite -#[cfg_attr(not(feature = "tflite"), doc("(not supported)"))] +#[cfg_attr(not(feature = "tflite"), doc = "(not supported)")] pub fn default_model_handler( _id: u32, meta: &ModelMetadata<'_>, From 93d939f209c7db02798fcc370b535644ba218078 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Jun 2022 05:21:57 +0800 Subject: [PATCH 75/84] Updating the path too the wit-files folder --- crates/runtime/src/zune/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs index 1c68971c02..7501538509 100644 --- a/crates/runtime/src/zune/mod.rs +++ b/crates/runtime/src/zune/mod.rs @@ -20,8 +20,8 @@ use crate::{ LoadError, }; -wit_bindgen_wasmer::export!("../../../wit-files/rune/runtime-v1.wit"); -wit_bindgen_wasmer::import!("../../../wit-files/rune/proc-block-v1.wit"); +wit_bindgen_wasmer::export!("../../wit-files/rune/runtime-v1.wit"); +wit_bindgen_wasmer::import!("../../wit-files/rune/proc-block-v1.wit"); pub(crate) trait Node { fn run(&mut self) -> Result<(), Error>; From 45c6dc6467b6f994169a6582556f1d39cf30c122 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Wed, 8 Jun 2022 23:34:40 +0530 Subject: [PATCH 76/84] Zune: Fix Output node handling for now --- crates/runtime/examples/zune_example.rs | 9 ++++++- crates/runtime/src/zune/mod.rs | 36 +++++++++++++++++++------ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/crates/runtime/examples/zune_example.rs b/crates/runtime/examples/zune_example.rs index 4e45230363..46019e560c 100644 --- a/crates/runtime/examples/zune_example.rs +++ b/crates/runtime/examples/zune_example.rs @@ -4,7 +4,7 @@ use hotg_rune_runtime::zune::{ElementType, TensorResult, ZuneEngine}; fn main() -> Result<(), Error> { let args: Vec = std::env::args().collect(); - let filename = args.get(1).map(|s| s.as_str()).unwrap_or("sine.rune"); + let filename = args.get(1).map(|s| s.as_str()).unwrap_or("/home/helios/Code/hotg/rune/crates/runtime/examples/sine.rune"); let sine_zune = std::fs::read(&filename) .with_context(|| format!("Unable to read \"{filename}\""))?; @@ -47,5 +47,12 @@ fn main() -> Result<(), Error> { zune_engine.get_output_tensor("sine", "Identity") ); + for node in zune_engine.output_nodes() { + let input_tensor_names = zune_engine.get_input_tensor_names(node)?; + for tensor_name in &input_tensor_names { + println!("Output {:?} {:?}: {:?}", node, tensor_name, zune_engine.get_input_tensor(node, tensor_name)); + } + } + Ok(()) } diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs index 7501538509..8b4d34e043 100644 --- a/crates/runtime/src/zune/mod.rs +++ b/crates/runtime/src/zune/mod.rs @@ -16,7 +16,7 @@ use zip; pub use self::{proc_block_v1::*, runtime_v1::*}; use crate::{ - zune::proc_block::{GraphContext, ProcBlockNode, TensorConstraint}, + zune::proc_block::{GraphContext, ProcBlockNode, TensorConstraint, Dimensions}, LoadError, }; @@ -156,7 +156,9 @@ impl ZuneEngine { let _span = tracing::debug_span!("Running Stage", %stage_name).entered(); - self.nodes.get_mut(stage_name).unwrap().run()?; + if let Some(node) = self.nodes.get_mut(stage_name) { + node.run()?; + } } Ok(()) } @@ -189,7 +191,7 @@ impl ZuneEngine { } pub fn get_input_tensor( - &mut self, + &self, node_name: &str, tensor_name: &str, ) -> Option { @@ -331,10 +333,9 @@ fn instantiate_nodes( shared_state: shared_state.clone(), }; - for item in pipeline { + for (stage_name, stage) in pipeline { // Collect each output tensor into tensors - let stage_name = item.0; - match item.1 { + match stage { // Models are handled on the host side, so we treat them separately Stage::Capability(stage) => { let wasm = @@ -390,8 +391,27 @@ fn instantiate_nodes( )?; nodes.insert(stage_name.to_string(), Box::new(pb)); }, - - _ => {}, // Do nothing for capabilities/outputs + Stage::Out(stage) => { + shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(stage_name) + .and_then(|c| { + for input in stage.inputs.iter() { + let tensor_key = key(&input.name, input.index); + let tensor_id = output_tensors.get(&tensor_key).copied(); + c.input_tensors.insert(tensor_key, + TensorConstraint { + tensor_id, + element_type: ElementType::U8, + dimensions: Dimensions::Dynamic + } + ); + } + Some(()) + }); + }, // Do nothing for capabilities/outputs } } From 3482c7c44b785073de99df85fc1f0c5070a05008 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Thu, 9 Jun 2022 19:20:48 +0530 Subject: [PATCH 77/84] Zune: Use IndexMap instead of HashMap for now --- crates/runtime/src/zune/mod.rs | 29 +++++++++++++-------------- crates/runtime/src/zune/proc_block.rs | 12 +++++------ crates/runtime/src/zune/tflite.rs | 15 +++++++------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs index 8b4d34e043..23e1bfc974 100644 --- a/crates/runtime/src/zune/mod.rs +++ b/crates/runtime/src/zune/mod.rs @@ -3,7 +3,6 @@ mod proc_block; mod tflite; use std::{ - collections::HashMap, fmt::{self, Display, Formatter}, io::{Cursor, Read}, sync::{Arc, Mutex}, @@ -36,13 +35,13 @@ pub(crate) struct Runtime { pub(crate) struct State { pub(crate) tensors: Vec>, pub(crate) tensor_constraints: Vec>, - pub(crate) graph_contexts: HashMap, + pub(crate) graph_contexts: IndexMap, } pub struct ZuneEngine { input_nodes: Vec, output_nodes: Vec, - nodes: HashMap>, + nodes: IndexMap>, processing_order: Vec, shared_state: Arc>, // resources } @@ -112,8 +111,8 @@ impl ZuneEngine { k.clone(), GraphContext { arguments, - input_tensors: HashMap::new(), - output_tensors: HashMap::new(), + input_tensors: IndexMap::new(), + output_tensors: IndexMap::new(), }, ) }) @@ -324,10 +323,10 @@ fn instantiate_nodes( pipeline: &IndexMap, mut read_zip_resource_by_path: impl FnMut(&str) -> Result, Error>, shared_state: &Arc>, - input_tensors: HashMap, - output_tensors: HashMap, -) -> Result>, Error> { - let mut nodes: HashMap> = HashMap::new(); + input_tensors: IndexMap, + output_tensors: IndexMap, +) -> Result>, Error> { + let mut nodes: IndexMap> = IndexMap::new(); let runtime = Runtime { shared_state: shared_state.clone(), @@ -424,8 +423,8 @@ fn load_model( stage_name: &str, stage: &ModelStage, shared_state: &Arc>, - input_tensors: &HashMap, - output_tensors: &HashMap, + input_tensors: &IndexMap, + output_tensors: &IndexMap, ) -> Result, Error> { match model_format { #[cfg(feature = "tflite")] @@ -452,8 +451,8 @@ fn get_tensors( ) -> Result< ( Vec>, - HashMap, - HashMap, + IndexMap, + IndexMap, Vec, ), Error, @@ -461,8 +460,8 @@ fn get_tensors( let mut nodes_to_visit = outputs.clone(); let mut nodes_visited = Vec::new(); let mut tensors: Vec> = Vec::new(); - let mut output_tensors: HashMap = HashMap::new(); - let mut input_tensors: HashMap = HashMap::new(); + let mut output_tensors: IndexMap = IndexMap::new(); + let mut input_tensors: IndexMap = IndexMap::new(); // For Inputs/Capabilities - We create an input so as to be able to inject inputs for item in inputs { diff --git a/crates/runtime/src/zune/proc_block.rs b/crates/runtime/src/zune/proc_block.rs index 982d6fe029..05e1af6f2d 100644 --- a/crates/runtime/src/zune/proc_block.rs +++ b/crates/runtime/src/zune/proc_block.rs @@ -1,7 +1,7 @@ use std::{ - collections::HashMap, sync::{Arc, Mutex}, }; +use indexmap::IndexMap; use anyhow::{Context, Error}; use wasmer::{ImportObject, Module, Store}; @@ -23,8 +23,8 @@ impl ProcBlockNode { node_id: &str, wasm: &[u8], runtime: &Runtime, - input_tensors: &HashMap, - output_tensors: &HashMap, + input_tensors: &IndexMap, + output_tensors: &IndexMap, ) -> Result { let shared_state = runtime.shared_state.clone(); let store = Store::default(); @@ -125,9 +125,9 @@ pub(crate) struct TensorConstraint { #[derive(Debug, Default, Clone)] pub(crate) struct GraphContext { - pub arguments: HashMap, - pub input_tensors: HashMap, - pub output_tensors: HashMap, + pub arguments: IndexMap, + pub input_tensors: IndexMap, + pub output_tensors: IndexMap, } impl runtime_v1::RuntimeV1 for Runtime { diff --git a/crates/runtime/src/zune/tflite.rs b/crates/runtime/src/zune/tflite.rs index b2ffe96ecb..62a27cc02d 100644 --- a/crates/runtime/src/zune/tflite.rs +++ b/crates/runtime/src/zune/tflite.rs @@ -1,9 +1,10 @@ use std::{ borrow::Cow, - collections::{HashMap, HashSet}, + collections::{HashSet}, sync::{Arc, Mutex}, }; +use indexmap::IndexMap; use anyhow::{Context, Error}; use hotg_rune_compiler::parse::ModelStage; use hotg_rune_core::TFLITE_MIMETYPE; @@ -41,8 +42,8 @@ impl ModelNode { node_data: &ModelStage, model_data: &[u8], shared_state: &Arc>, - input_tensors: &HashMap, - output_tensors: &HashMap, + input_tensors: &IndexMap, + output_tensors: &IndexMap, ) -> Result { // Create Inference Context let context = InferenceContext::create_context( @@ -89,14 +90,14 @@ impl ModelNode { model_tensors: &mut dyn Iterator< Item = RuneCoralTensorDescriptor, >, - pipeline_tensors: &HashMap| + pipeline_tensors: &IndexMap| -> Result< - (HashSet, HashMap), + (HashSet, IndexMap), Error, > { let mut tensor_indices: HashSet = HashSet::new(); - let mut tensor_constraints: HashMap = - HashMap::new(); + let mut tensor_constraints: IndexMap = + IndexMap::new(); let mut i = 0; let mut s = shared_state.lock().unwrap(); From 3f87a7ac4b638965dcec25dc13e7252cd431277a Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 13 Jun 2022 18:36:46 +0800 Subject: [PATCH 78/84] Added a manual Deserialize implementation to Stage for better error messages --- crates/compiler/src/parse/yaml.rs | 45 ++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index 10dd257992..4f5b13fe5d 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -363,15 +363,7 @@ pub struct OutStage { } /// A stage in the Rune's pipeline. -#[derive( - Debug, - Clone, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, - JsonSchema, -)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, JsonSchema)] #[serde(untagged, rename_all = "kebab-case")] pub enum Stage { Model(ModelStage), @@ -435,6 +427,41 @@ impl Stage { } } +impl<'de> Deserialize<'de> for Stage { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = serde_yaml::Mapping::deserialize(deserializer)?; + + if value.contains_key(&"capability".into()) { + return serde_yaml::from_value(serde_yaml::Value::Mapping(value)) + .map(Stage::Capability) + .map_err(D::Error::custom); + } + + if value.contains_key(&"proc-block".into()) { + return serde_yaml::from_value(serde_yaml::Value::Mapping(value)) + .map(Stage::ProcBlock) + .map_err(D::Error::custom); + } + + if value.contains_key(&"model".into()) { + return serde_yaml::from_value(serde_yaml::Value::Mapping(value)) + .map(Stage::Model) + .map_err(D::Error::custom); + } + + if value.contains_key(&"out".into()) { + return serde_yaml::from_value(serde_yaml::Value::Mapping(value)) + .map(Stage::Out) + .map_err(D::Error::custom); + } + + Err(D::Error::custom("The value didn't parse as a capability, model, proc-block, or output")) + } +} + /// Something that could be either a reference to a resource (`$resource`) /// or a plain string (`./path`). #[derive(Debug, Clone, PartialEq, Eq, Hash)] From 88048cf480d11d897fd922653147d3f8c4ecd939 Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Mon, 13 Jun 2022 17:21:52 +0530 Subject: [PATCH 79/84] Zune: Error out when proc_block loading fails --- crates/runtime/src/zune/proc_block.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/runtime/src/zune/proc_block.rs b/crates/runtime/src/zune/proc_block.rs index 05e1af6f2d..0884499fd3 100644 --- a/crates/runtime/src/zune/proc_block.rs +++ b/crates/runtime/src/zune/proc_block.rs @@ -37,7 +37,7 @@ impl ProcBlockNode { ProcBlockV1::instantiate(&store, &module, &mut imports) .context("Unable to instantiate the WebAssembly module")?; - let _result = pb.graph(node_id); + let _result = pb.graph(node_id)??; // Assign tensors // TODO: See if this can be more smart. From 1b863693bfd46cc25fd91e6fc268b5f776170efa Mon Sep 17 00:00:00 2001 From: Dinesh Manajipet Date: Mon, 13 Jun 2022 17:22:06 +0530 Subject: [PATCH 80/84] Zune: Minor reafactoring in mod.rs --- crates/runtime/src/zune/mod.rs | 161 +++++++++++++++++---------------- 1 file changed, 83 insertions(+), 78 deletions(-) diff --git a/crates/runtime/src/zune/mod.rs b/crates/runtime/src/zune/mod.rs index 23e1bfc974..d19c140350 100644 --- a/crates/runtime/src/zune/mod.rs +++ b/crates/runtime/src/zune/mod.rs @@ -334,89 +334,94 @@ fn instantiate_nodes( for (stage_name, stage) in pipeline { // Collect each output tensor into tensors - match stage { - // Models are handled on the host side, so we treat them separately - Stage::Capability(stage) => { - let wasm = - read_zip_resource_by_path(&stage.capability.to_string()) - .context("Unable to load the capability")?; - - let pb = ProcBlockNode::load( - &stage_name, - &wasm, - &runtime, - &input_tensors, - &output_tensors, - )?; - nodes.insert(stage_name.to_string(), Box::new(pb)); - }, - Stage::Model(stage) => { - // Instantiating the model's inference context here because that - // way model_data gets deallocated once we are done with it - // This way memory usage is under control - let model_data = - read_zip_resource_by_path(&stage.model.to_string()) - .with_context(|| { - format!( - "Unable to read model from zune {}", - stage.model - ) - })?; - - let model_format = - stage.args.get("model-format").map(|f| f.to_string()); - let node = load_model( - &model_data, - model_format.as_deref().unwrap_or("tflite"), - stage_name, - stage, - shared_state, - &input_tensors, - &output_tensors, - )?; - nodes.insert(stage_name.to_string(), node); - }, - Stage::ProcBlock(stage) => { - let wasm = - read_zip_resource_by_path(&stage.proc_block.to_string()) - .context("Unable to load the proc_block")?; - - let pb = ProcBlockNode::load( - &stage_name, - &wasm, - &runtime, - &input_tensors, - &output_tensors, - )?; - nodes.insert(stage_name.to_string(), Box::new(pb)); - }, - Stage::Out(stage) => { - shared_state - .lock() - .unwrap() - .graph_contexts - .get_mut(stage_name) - .and_then(|c| { - for input in stage.inputs.iter() { - let tensor_key = key(&input.name, input.index); - let tensor_id = output_tensors.get(&tensor_key).copied(); - c.input_tensors.insert(tensor_key, - TensorConstraint { - tensor_id, - element_type: ElementType::U8, - dimensions: Dimensions::Dynamic - } - ); - } - Some(()) - }); - }, // Do nothing for capabilities/outputs - } + instantiate_node(stage, &mut read_zip_resource_by_path, stage_name, &runtime, &input_tensors, &output_tensors, &mut nodes, shared_state) + .with_context(|| format!("Unable to load node \"{stage_name}\""))?; } Ok(nodes) } +fn instantiate_node(stage: &Stage, read_zip_resource_by_path: &mut impl FnMut(&str) -> Result, Error>, stage_name: &String, runtime: &Runtime, input_tensors: &IndexMap, output_tensors: &IndexMap, nodes: &mut IndexMap>, shared_state: &Arc>) -> Result<(), Error> { + Ok(match stage { + // Models are handled on the host side, so we treat them separately + Stage::Capability(stage) => { + let wasm = + read_zip_resource_by_path(&stage.capability.to_string()) + .context("Unable to load the capability")?; + + let pb = ProcBlockNode::load( + &stage_name, + &wasm, + runtime, + input_tensors, + output_tensors, + )?; + nodes.insert(stage_name.to_string(), Box::new(pb)); + }, + Stage::Model(stage) => { + // Instantiating the model's inference context here because that + // way model_data gets deallocated once we are done with it + // This way memory usage is under control + let model_data = + read_zip_resource_by_path(&stage.model.to_string()) + .with_context(|| { + format!( + "Unable to read model from zune {}", + stage.model + ) + })?; + + let model_format = + stage.args.get("model-format").map(|f| f.to_string()); + let node = load_model( + &model_data, + model_format.as_deref().unwrap_or("tflite"), + stage_name, + stage, + shared_state, + input_tensors, + output_tensors, + )?; + nodes.insert(stage_name.to_string(), node); + }, + Stage::ProcBlock(stage) => { + let wasm = + read_zip_resource_by_path(&stage.proc_block.to_string()) + .context("Unable to load the proc_block")?; + + let pb = ProcBlockNode::load( + &stage_name, + &wasm, + runtime, + input_tensors, + output_tensors, + )?; + nodes.insert(stage_name.to_string(), Box::new(pb)); + }, + Stage::Out(stage) => { + shared_state + .lock() + .unwrap() + .graph_contexts + .get_mut(stage_name) + .and_then(|c| { + for input in stage.inputs.iter() { + let tensor_key = key(&input.name, input.index); + let tensor_id = output_tensors.get(&tensor_key).copied(); + c.input_tensors.insert(tensor_key, + TensorConstraint { + tensor_id, + element_type: ElementType::U8, + dimensions: Dimensions::Dynamic + } + ); + } + Some(()) + }); + }, // Do nothing for capabilities/outputs + }) +} + fn load_model( model_data: &[u8], model_format: &str, From 84f2f02a287662d0a407909d2a136338c3195177 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 5 Jul 2022 10:32:50 +0800 Subject: [PATCH 81/84] Treat a URI with invalid characters as a file path --- crates/compiler/src/parse/yaml.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index 4f5b13fe5d..a16514d743 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -21,7 +21,7 @@ use serde::{ de::{Deserialize, Deserializer, Error as _}, ser::{Serialize, Serializer}, }; -use uriparse::{URIError, URI}; +use uriparse::{URIError, URI, PathError}; static RESOURCE_NAME_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^\$[_a-zA-Z][_a-zA-Z0-9]*$").unwrap()); @@ -263,7 +263,7 @@ impl FromStr for Path { match URI::try_from(s) { Ok(u) => Ok(Path::Uri(u.into_owned())), - Err(URIError::NotURI) => Ok(Path::FileSystem(s.to_string())), + Err(URIError::NotURI) | Err(URIError::Path(PathError::InvalidCharacter)) => Ok(Path::FileSystem(s.to_string())), Err(e) => Err(e), } } From 5efd96fcb9b99c2277405fcfd10e7945259df127 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 5 Jul 2022 10:49:47 +0800 Subject: [PATCH 82/84] Added a test for paths containing a space --- crates/compiler/src/parse/yaml.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index a16514d743..030d383308 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -1253,4 +1253,13 @@ pipeline: .validate(&number) .unwrap_or_else(|e| handle_errors(e)); } + + #[test] + fn parse_paths_containing_a_space() { + let path = "/path/to/folder/with a/space"; + + let got: Path = path.parse().unwrap(); + + assert_eq!(got, Path::FileSystem(path.to_string())); + } } From 9fffed1af667cedc2fca1339b2db5e7f633f6a5c Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 5 Jul 2022 19:57:02 +0800 Subject: [PATCH 83/84] Made sure we can convert a FileSystem path into a URI --- Cargo.lock | 1 + crates/compiler/Cargo.toml | 1 + crates/compiler/src/config.rs | 10 +++++ crates/compiler/src/parse/query.rs | 72 ++++++++++++++++++++++++------ crates/compiler/src/parse/yaml.rs | 19 ++++---- 5 files changed, 82 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc913e3c12..df57b2f6fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,6 +1471,7 @@ dependencies = [ "insta", "jsonschema", "once_cell", + "percent-encoding 2.1.0", "pretty_assertions 1.2.1", "queryst", "regex 1.5.6", diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index c29a64cbf6..b235263077 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -12,6 +12,7 @@ readme = "README.md" graphql_client = { version = "0.10.0", features = ["reqwest-blocking"], optional = true } indexmap = { version = "1.8.1", features = ["serde-1"] } once_cell = "1.10.0" +percent-encoding = "2.1.0" queryst = { version = "2.1.0" } regex = "1.5.5" reqwest = { version = "0.11.10", features = ["blocking"], optional = true } diff --git a/crates/compiler/src/config.rs b/crates/compiler/src/config.rs index 9cacfdfaef..bc37e08f88 100644 --- a/crates/compiler/src/config.rs +++ b/crates/compiler/src/config.rs @@ -8,6 +8,16 @@ pub struct BuildConfig { pub features: FeatureFlags, } +impl Default for BuildConfig { + fn default() -> Self { + Self { + current_directory: std::env::current_dir() + .expect("Unable to determine the current directory"), + features: FeatureFlags::stable(), + } + } +} + /// Flags used by the Rune compiler to enable features. #[derive( Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index 299973ca1a..af460e99ef 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -267,12 +267,15 @@ fn model_files( } fn read(db: &dyn Frontend, path: &Path) -> Result, crate::Error> { - file_uri(db, path) + path_to_uri(db, path) .map_err(|e| Arc::new(e) as crate::Error) .and_then(|uri| db.read(&uri).map_err(|e| Arc::new(e) as crate::Error)) } -fn file_uri(db: &dyn Frontend, path: &Path) -> Result, URIError> { +fn path_to_uri( + db: &dyn Frontend, + path: &Path, +) -> Result, URIError> { match path { Path::WellKnown(w) => Ok(wapm_uri(*w)), Path::Uri(u) => Ok(u.to_owned()), @@ -280,21 +283,46 @@ fn file_uri(db: &dyn Frontend, path: &Path) -> Result, URIError> { let BuildConfig { current_directory, .. } = db.config(); - let full_path = current_directory - .join(path) - .display() - .to_string() - .replace('\\', "/"); - let path = uriparse::Path::try_from(full_path.as_str())?; - Ok(URIBuilder::new() - .with_scheme(uriparse::Scheme::File) - .with_path(path) - .build()? - .into_owned()) + let full_path = current_directory.join(path); + + file_uri(&full_path) }, } } +fn file_uri(path: &std::path::Path) -> Result, URIError> { + let segments: Vec<_> = path + .components() + .filter_map(|segment| match segment { + std::path::Component::Normal(segment) => segment.to_str(), + std::path::Component::ParentDir => Some(".."), + std::path::Component::CurDir => None, + std::path::Component::Prefix(_) => None, + std::path::Component::RootDir => None, + }) + .map(|s| { + percent_encoding::utf8_percent_encode( + &s, + percent_encoding::NON_ALPHANUMERIC, + ) + .to_string() + }) + .collect(); + + let mut joined = segments.join("/"); + if path.is_absolute() { + joined.insert(0, '/'); + } + + let path = uriparse::Path::try_from(joined.as_str())?; + let mut builder = URIBuilder::new() + .with_scheme(uriparse::Scheme::File) + .with_path(path); + builder.try_authority(Some(""))?; + + builder.build().map(|u| u.into_owned()) +} + fn wapm_uri(w: WellKnownPath) -> URI<'static> { let uri = match w { WellKnownPath::Accel => { @@ -307,3 +335,21 @@ fn wapm_uri(w: WellKnownPath) -> URI<'static> { uri.try_into().expect("Should never fail") } + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn convert_filename_with_space_to_uri() { + let path = PathBuf::from("/path/to/folder/with a/space"); + + let got = file_uri(&path).unwrap(); + + assert_eq!( + got, + URI::try_from("file:///path/to/folder/with%20a/space").unwrap() + ); + } +} diff --git a/crates/compiler/src/parse/yaml.rs b/crates/compiler/src/parse/yaml.rs index 030d383308..cdd3540ccc 100644 --- a/crates/compiler/src/parse/yaml.rs +++ b/crates/compiler/src/parse/yaml.rs @@ -21,7 +21,7 @@ use serde::{ de::{Deserialize, Deserializer, Error as _}, ser::{Serialize, Serializer}, }; -use uriparse::{URIError, URI, PathError}; +use uriparse::{PathError, URIError, URI}; static RESOURCE_NAME_PATTERN: Lazy = Lazy::new(|| Regex::new(r"^\$[_a-zA-Z][_a-zA-Z0-9]*$").unwrap()); @@ -263,7 +263,10 @@ impl FromStr for Path { match URI::try_from(s) { Ok(u) => Ok(Path::Uri(u.into_owned())), - Err(URIError::NotURI) | Err(URIError::Path(PathError::InvalidCharacter)) => Ok(Path::FileSystem(s.to_string())), + Err(URIError::NotURI) + | Err(URIError::Path(PathError::InvalidCharacter)) => { + Ok(Path::FileSystem(s.to_string())) + }, Err(e) => Err(e), } } @@ -1254,12 +1257,12 @@ pipeline: .unwrap_or_else(|e| handle_errors(e)); } - #[test] - fn parse_paths_containing_a_space() { - let path = "/path/to/folder/with a/space"; + #[test] + fn parse_paths_containing_a_space() { + let path = "/path/to/folder/with a/space"; - let got: Path = path.parse().unwrap(); + let got: Path = path.parse().unwrap(); - assert_eq!(got, Path::FileSystem(path.to_string())); - } + assert_eq!(got, Path::FileSystem(path.to_string())); + } } From 8a12dda1be6bc407505c6f2d89339dd1e78b41c0 Mon Sep 17 00:00:00 2001 From: Michael Bryan Date: Tue, 5 Jul 2022 20:24:52 +0800 Subject: [PATCH 84/84] Made sure the default asset loader can handle percent-encoded paths --- Cargo.lock | 1 + crates/compiler/Cargo.toml | 1 + crates/compiler/src/asset_loader/builtin.rs | 29 ++++++++++- crates/compiler/src/asset_loader/mod.rs | 53 +++++++++++++++++++- crates/compiler/src/im.rs | 32 +++++++++++- crates/compiler/src/parse/query.rs | 55 +-------------------- 6 files changed, 113 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df57b2f6fb..38e0cbea5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1481,6 +1481,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "tempfile", "thiserror", "tracing", "tracing-subscriber", diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml index b235263077..15bef4e03e 100644 --- a/crates/compiler/Cargo.toml +++ b/crates/compiler/Cargo.toml @@ -31,6 +31,7 @@ insta = "1.14.0" jsonschema = "0.15.2" pretty_assertions = "1.2.1" reqwest = "0.11.10" +tempfile = "3.3.0" tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } tracing-test = "0.2.1" diff --git a/crates/compiler/src/asset_loader/builtin.rs b/crates/compiler/src/asset_loader/builtin.rs index 13ceb11e4f..491e265e02 100644 --- a/crates/compiler/src/asset_loader/builtin.rs +++ b/crates/compiler/src/asset_loader/builtin.rs @@ -1,6 +1,7 @@ use std::path::{Component, Path, PathBuf}; use graphql_client::{GraphQLQuery, Response}; +use percent_encoding::percent_decode_str; use reqwest::blocking::Client; use uriparse::{Scheme, URI}; @@ -49,7 +50,7 @@ impl Default for DefaultAssetLoader { } impl AssetLoader for DefaultAssetLoader { - #[tracing::instrument(skip(self), err)] + #[tracing::instrument(skip_all, err, fields(uri = %uri))] fn read(&self, uri: &URI<'_>) -> Result, ReadError> { match uri.scheme() { Scheme::HTTP | Scheme::HTTPS => { @@ -183,7 +184,8 @@ fn local_file( }; for segment in uri.segments() { - path.push(segment.as_str()); + let decoded = percent_decode_str(segment.as_str()).decode_utf8_lossy(); + path.push(decoded.as_ref()); } tracing::debug!(path = %path.display(), "Reading a file from disk"); @@ -201,3 +203,26 @@ impl From for ReadError { } } } + +#[cfg(test)] +mod tests { + use tempfile::tempdir; + + use super::*; + + #[test] + fn read_file_with_a_percent_encoded_uri() { + let loader = DefaultAssetLoader::default(); + let temp = tempdir().unwrap(); + let file_name = + temp.path().join("folder").join("with a").join("space.txt"); + std::fs::create_dir_all(file_name.parent().unwrap()).unwrap(); + std::fs::write(&file_name, b"Hello, World!").unwrap(); + let uri = crate::asset_loader::file_uri(&file_name).unwrap(); + assert!(uri.to_string().contains("%20")); + + let data = loader.read(&uri).unwrap(); + + assert_eq!(data, *b"Hello, World!"); + } +} diff --git a/crates/compiler/src/asset_loader/mod.rs b/crates/compiler/src/asset_loader/mod.rs index 96fa0011dd..ce7ad26a39 100644 --- a/crates/compiler/src/asset_loader/mod.rs +++ b/crates/compiler/src/asset_loader/mod.rs @@ -12,7 +12,7 @@ use std::{ }; use serde::Deserialize; -use uriparse::{Scheme, URI}; +use uriparse::{Scheme, URIBuilder, URIError, URI}; use crate::{im::Vector, Text}; @@ -112,7 +112,7 @@ impl Cached { } impl AssetLoader for Cached { - #[tracing::instrument(skip(self), err)] + #[tracing::instrument(skip_all, err, fields(uri = %uri))] fn read(&self, uri: &URI<'_>) -> Result, ReadError> { if let Some(cached_value) = self.cache.read().ok().and_then(|c| c.get(uri).cloned()) @@ -319,8 +319,45 @@ pub enum ParseWapmUriError { }, } +pub(crate) fn file_uri( + path: &std::path::Path, +) -> Result, URIError> { + let segments: Vec<_> = path + .components() + .filter_map(|segment| match segment { + std::path::Component::Normal(segment) => segment.to_str(), + std::path::Component::ParentDir => Some(".."), + std::path::Component::CurDir => None, + std::path::Component::Prefix(_) => None, + std::path::Component::RootDir => None, + }) + .map(|s| { + percent_encoding::utf8_percent_encode( + &s, + percent_encoding::NON_ALPHANUMERIC, + ) + .to_string() + }) + .collect(); + + let mut joined = segments.join("/"); + if path.is_absolute() { + joined.insert(0, '/'); + } + + let path = uriparse::Path::try_from(joined.as_str())?; + let mut builder = URIBuilder::new() + .with_scheme(uriparse::Scheme::File) + .with_path(path); + builder.try_authority(Some(""))?; + + builder.build().map(|u| u.into_owned()) +} + #[cfg(test)] mod tests { + use std::path::PathBuf; + use super::*; #[test] @@ -398,4 +435,16 @@ mod tests { assert_eq!(round_tripped, uri); } } + + #[test] + fn convert_filename_with_space_to_uri() { + let path = PathBuf::from("/path/to/folder/with a/space"); + + let got = file_uri(&path).unwrap(); + + assert_eq!( + got, + URI::try_from("file:///path/to/folder/with%20a/space").unwrap() + ); + } } diff --git a/crates/compiler/src/im.rs b/crates/compiler/src/im.rs index 9116b9537b..4d71d58064 100644 --- a/crates/compiler/src/im.rs +++ b/crates/compiler/src/im.rs @@ -96,9 +96,9 @@ impl serde::Serialize for Text { #[derive( Debug, - PartialEq, Eq, Hash, + PartialEq, PartialOrd, Ord, serde::Serialize, @@ -160,6 +160,36 @@ impl AsRef<[A]> for Vector { } } +impl PartialEq<[A]> for Vector { + fn eq(&self, other: &[A]) -> bool { + self.as_ref() == other + } +} + +impl PartialEq<[A; N]> for Vector { + fn eq(&self, other: &[A; N]) -> bool { + self.as_ref() == other + } +} + +impl PartialEq> for [A] { + fn eq(&self, other: &Vector) -> bool { + other.eq(self) + } +} + +impl PartialEq> for [A; N] { + fn eq(&self, other: &Vector) -> bool { + other.as_ref() == &self[..] + } +} + +impl PartialEq> for &'_ [A; N] { + fn eq(&self, other: &Vector) -> bool { + other.as_ref() == &self[..] + } +} + #[derive( Debug, PartialEq, diff --git a/crates/compiler/src/parse/query.rs b/crates/compiler/src/parse/query.rs index af460e99ef..e9b2235794 100644 --- a/crates/compiler/src/parse/query.rs +++ b/crates/compiler/src/parse/query.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use uriparse::{URIBuilder, URIError, URI}; +use uriparse::{URIError, URI}; use crate::{ asset_loader::AssetLoader, @@ -285,44 +285,11 @@ fn path_to_uri( } = db.config(); let full_path = current_directory.join(path); - file_uri(&full_path) + crate::asset_loader::file_uri(&full_path) }, } } -fn file_uri(path: &std::path::Path) -> Result, URIError> { - let segments: Vec<_> = path - .components() - .filter_map(|segment| match segment { - std::path::Component::Normal(segment) => segment.to_str(), - std::path::Component::ParentDir => Some(".."), - std::path::Component::CurDir => None, - std::path::Component::Prefix(_) => None, - std::path::Component::RootDir => None, - }) - .map(|s| { - percent_encoding::utf8_percent_encode( - &s, - percent_encoding::NON_ALPHANUMERIC, - ) - .to_string() - }) - .collect(); - - let mut joined = segments.join("/"); - if path.is_absolute() { - joined.insert(0, '/'); - } - - let path = uriparse::Path::try_from(joined.as_str())?; - let mut builder = URIBuilder::new() - .with_scheme(uriparse::Scheme::File) - .with_path(path); - builder.try_authority(Some(""))?; - - builder.build().map(|u| u.into_owned()) -} - fn wapm_uri(w: WellKnownPath) -> URI<'static> { let uri = match w { WellKnownPath::Accel => { @@ -335,21 +302,3 @@ fn wapm_uri(w: WellKnownPath) -> URI<'static> { uri.try_into().expect("Should never fail") } - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - #[test] - fn convert_filename_with_space_to_uri() { - let path = PathBuf::from("/path/to/folder/with a/space"); - - let got = file_uri(&path).unwrap(); - - assert_eq!( - got, - URI::try_from("file:///path/to/folder/with%20a/space").unwrap() - ); - } -}