From 13c6b8c40692730c5ca38f4ec8edb738071073c2 Mon Sep 17 00:00:00 2001 From: sealmove Date: Fri, 2 Sep 2022 13:20:35 +0300 Subject: [PATCH 001/163] update docs since go sdk changed executor target (wagi -> spin) Signed-off-by: sealmove --- docs/content/go-components.md | 2 +- docs/content/http-trigger.md | 2 +- examples/http-tinygo-outbound-http/readme.md | 1 - sdk/go/readme.md | 19 +++++++++++++------ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/content/go-components.md b/docs/content/go-components.md index d1e365b408..183d8f86df 100644 --- a/docs/content/go-components.md +++ b/docs/content/go-components.md @@ -132,7 +132,7 @@ allowed_http_hosts = [ "https://some-random-api.ml" ] route = "/hello" ``` -> Spin HTTP components written in Go must currently use the Wagi executor. +> Spin HTTP components written in Go use the Spin executor. Running the application using `spin up --file spin.toml` will start the HTTP listener locally (by default on `localhost:3000`), and our component can diff --git a/docs/content/http-trigger.md b/docs/content/http-trigger.md index 454d835794..bdbbfdaea6 100644 --- a/docs/content/http-trigger.md +++ b/docs/content/http-trigger.md @@ -214,7 +214,7 @@ print("content-type: text/html; charset=UTF-8\n\n"); print("hello world\n"); ``` -The [Go SDK for Spin](./go-components.md) is built on the Wagi executor support. +The [Go SDK for Spin](./go-components.md) supports the Spin executor. Here is another example, written in [Grain](https://grain-lang.org/), a new programming language that natively targets WebAssembly: diff --git a/examples/http-tinygo-outbound-http/readme.md b/examples/http-tinygo-outbound-http/readme.md index e270c64e1b..6d9d5a4060 100644 --- a/examples/http-tinygo-outbound-http/readme.md +++ b/examples/http-tinygo-outbound-http/readme.md @@ -42,7 +42,6 @@ source = "main.wasm" allowed_http_hosts = [ "https://some-random-api.ml", "https://postman-echo.com" ] [component.trigger] route = "/hello" -executor = { type = "wagi" } ``` At this point, we can execute the application with the `spin` CLI: diff --git a/sdk/go/readme.md b/sdk/go/readme.md index c26b688660..9bf962ee46 100644 --- a/sdk/go/readme.md +++ b/sdk/go/readme.md @@ -1,14 +1,21 @@ # The (Tiny)Go SDK for Spin This package contains an SDK that facilitates building Spin components in -(Tiny)Go. It currently allows building HTTP components that target the Wagi +(Tiny)Go. It allows building HTTP components that target the Spin executor. ```go -func main() { - // call the HandleRequest function - spin_http.HandleRequest(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "Hello, Fermyon!") - }) +import ( + "fmt" + spinhttp "github.com/fermyon/spin/sdk/go/http" +) + +func init() { + // call the Handle function + spinhttp.Handle(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, Fermyon!") + }) } + +func main() {} ``` From 3e6e130ff7cd8779130fa955cc8cf3f3e8938fff Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Mon, 8 Aug 2022 12:15:19 -0700 Subject: [PATCH 002/163] feat: add support for spin plugins Signed-off-by: Kate Goldenring Co-authored-by: karthik Ganeshram --- Cargo.lock | 30 ++++ Cargo.toml | 2 + crates/plugins/Cargo.toml | 22 +++ crates/plugins/src/git.rs | 69 +++++++++ crates/plugins/src/install.rs | 213 ++++++++++++++++++++++++++ crates/plugins/src/lib.rs | 5 + crates/plugins/src/plugin_manifest.rs | 136 ++++++++++++++++ crates/plugins/src/prompt.rs | 61 ++++++++ src/bin/spin.rs | 10 +- src/commands.rs | 4 + src/commands/external.rs | 26 ++++ src/commands/plugins.rs | 124 +++++++++++++++ 12 files changed, 700 insertions(+), 2 deletions(-) create mode 100644 crates/plugins/Cargo.toml create mode 100644 crates/plugins/src/git.rs create mode 100644 crates/plugins/src/install.rs create mode 100644 crates/plugins/src/lib.rs create mode 100644 crates/plugins/src/plugin_manifest.rs create mode 100644 crates/plugins/src/prompt.rs create mode 100644 src/commands/external.rs create mode 100644 src/commands/plugins.rs diff --git a/Cargo.lock b/Cargo.lock index 3e7f0028dc..d4fed602f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3310,6 +3310,7 @@ dependencies = [ "spin-http-engine", "spin-loader", "spin-manifest", + "spin-plugins", "spin-publish", "spin-redis-engine", "spin-templates", @@ -3460,6 +3461,24 @@ dependencies = [ "url", ] +[[package]] +name = "spin-plugins" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "flate2", + "log", + "reqwest", + "serde", + "serde_json", + "sha2 0.10.3", + "tar", + "tempfile", + "tokio", + "url", +] + [[package]] name = "spin-publish" version = "0.2.0" @@ -3716,6 +3735,17 @@ dependencies = [ "winx 0.33.0", ] +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index ae1d9c0f3d..11f2226b92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ spin-engine = { path = "crates/engine" } spin-http-engine = { path = "crates/http" } spin-loader = { path = "crates/loader" } spin-manifest = { path = "crates/manifest" } +spin-plugins = { path = "crates/plugins" } spin-publish = { path = "crates/publish" } spin-redis-engine = { path = "crates/redis" } spin-templates = { path = "crates/templates" } @@ -78,6 +79,7 @@ members = [ "crates/manifest", "crates/outbound-http", "crates/outbound-redis", + "crates/plugins", "crates/redis", "crates/templates", "crates/testing", diff --git a/crates/plugins/Cargo.toml b/crates/plugins/Cargo.toml new file mode 100644 index 0000000000..c25f3035b7 --- /dev/null +++ b/crates/plugins/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "spin-plugins" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0" +bytes = "1.1" +flate2 = "1.0" +log = { version = "0.4", default-features = false } +reqwest = { version = "0.11", features = ["json"] } +sha2 = "0.10.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tar = "0.4.38" +tempfile = "3.3.0" +tokio = { version = "1.10", features = [ "fs", "process", "rt", "macros" ] } +url = "2.2.2" + + diff --git a/crates/plugins/src/git.rs b/crates/plugins/src/git.rs new file mode 100644 index 0000000000..6a13182125 --- /dev/null +++ b/crates/plugins/src/git.rs @@ -0,0 +1,69 @@ +use anyhow::{anyhow, Result}; +use std::path::PathBuf; +use tokio::process::Command; +use url::Url; + +const PLUGINS_REPO_BRANCH: &str = "main"; + +pub struct GitSource { + source_url: Url, + branch: String, + local_repo_dir: PathBuf, +} + +impl GitSource { + pub fn new( + source_url: &Url, + branch: Option, + local_repo_dir: PathBuf, + ) -> Result { + Ok(Self { + source_url: source_url.clone(), + branch: branch.unwrap_or_else(|| PLUGINS_REPO_BRANCH.to_owned()), + local_repo_dir, + }) + } + + pub async fn clone(&self) -> Result<()> { + let mut git = Command::new("git"); + git.args([ + "clone", + &self.source_url.to_string(), + "--branch", + &self.branch, + "--single-branch", + &self.local_repo_dir.to_string_lossy(), + ]); + let clone_result = git.output().await?; + match clone_result.status.success() { + true => { + println!("Cloned Repository Successfully!"); + Ok(()) + } + false => Err(anyhow!( + "Error cloning Git repo {}: {}", + self.source_url, + String::from_utf8(clone_result.stderr) + .unwrap_or_else(|_| "(cannot get error)".to_owned()) + )), + } + } + + pub async fn pull(&self) -> Result<()> { + let mut git = Command::new("git"); + git.args(["-C", &self.local_repo_dir.to_string_lossy(), "pull"]); + let pull_result = git.output().await?; + match pull_result.status.success() { + true => { + println!("Updated Repository Successfully"); + Ok(()) + } + false => Err(anyhow!( + "Error updating Git repo at {:?}: {}", + self.local_repo_dir, + String::from_utf8(pull_result.stderr) + .unwrap_or_else(|_| "(cannot update error)".to_owned()) + )), + } + } +} diff --git a/crates/plugins/src/install.rs b/crates/plugins/src/install.rs new file mode 100644 index 0000000000..3fd60584e9 --- /dev/null +++ b/crates/plugins/src/install.rs @@ -0,0 +1,213 @@ +use super::git::GitSource; +use super::plugin_manifest::{Os, PluginManifest}; +use super::prompt::Prompter; +use anyhow::{anyhow, Result}; +use flate2::read::GzDecoder; +use std::{ + fs::{self, File}, + io::{copy, Cursor}, + path::{Path, PathBuf}, +}; +use tar::Archive; +use tempfile::{tempdir, TempDir}; +use url::Url; + +/// Name of the subdirectory that contains the installed plugin JSON manifests +const PLUGIN_MANIFESTS_DIRECTORY_NAME: &str = "manifests"; +const PLUGINS_REPO_LOCAL_DIRECTORY: &str = ".spin-plugins"; +const PLUGINS_REPO_MANIFESTS_DIRECTORY: &str = "manifests"; + +pub enum ManifestLocation { + Local(PathBuf), + Remote(Url), + PluginsRepository(PluginInfo), +} + +pub struct PluginInfo { + name: String, + repo_url: Url, + // version +} +impl PluginInfo { + pub fn new(name: String, repo_url: Url) -> Self { + Self { name, repo_url } + } +} + +pub struct PluginInstaller { + manifest_location: ManifestLocation, + plugins_dir: PathBuf, + yes_to_all: bool, +} + +impl PluginInstaller { + pub fn new( + manifest_location: ManifestLocation, + plugins_dir: PathBuf, + yes_to_all: bool, + ) -> Self { + Self { + manifest_location, + plugins_dir, + yes_to_all, + } + } + + pub async fn install(&self) -> Result<()> { + // TODO: Potentially handle errors to give useful error messages + let plugin_manifest: PluginManifest = match &self.manifest_location { + ManifestLocation::Remote(url) => { + // Remote manifest source is provided + log::info!("Pulling manifest for plugin from {}", url); + reqwest::get(url.as_ref()) + .await? + .json::() + .await? + } + ManifestLocation::Local(path) => { + // Local manifest source is provided + log::info!("Pulling manifest for plugin from {:?}", path); + let file = File::open(path)?; + serde_json::from_reader(file)? + } + ManifestLocation::PluginsRepository(info) => { + log::info!( + "Pulling manifest for plugin {} from {}", + info.name, + info.repo_url + ); + let git_source = GitSource::new( + &info.repo_url, + None, + self.plugins_dir.join(PLUGINS_REPO_LOCAL_DIRECTORY), + )?; + if !self + .plugins_dir + .join(PLUGINS_REPO_LOCAL_DIRECTORY) + .join(".git") + .exists() + { + git_source.clone().await?; + // self.get_latest_plugin_repo(&info.repo_url)?; + } else { + git_source.pull().await?; + // self.update_plugins_repository()?; + } + let file = File::open( + &self + .plugins_dir + .join(PLUGINS_REPO_LOCAL_DIRECTORY) + .join(PLUGINS_REPO_MANIFESTS_DIRECTORY) + .join(&info.name) + .join(get_manifest_file_name(&info.name)), + )?; + serde_json::from_reader(file)? + } + }; + + let os: Os = if cfg!(target_os = "windows") { + Os::Windows + } else if cfg!(target_os = "linux") { + Os::Linux + } else if cfg!(target_os = "macos") { + Os::Osx + } else { + return Err(anyhow!("This plugin is not supported on this OS")); + }; + // TODO: Add logic for architecture as well + let plugin_package = plugin_manifest + .packages + .iter() + .find(|p| p.os == os) + .ok_or_else(|| anyhow!("This plugin does not support this OS"))?; + let target_url = plugin_package.url.to_owned(); + + // Ask for user confirmation if not overridden with CLI option + if !self.yes_to_all + && !Prompter::new(&plugin_manifest.name, &plugin_manifest.license, &target_url)? + .run()? + { + // User has requested to not install package, returning early + println!("Plugin {} will not be installed", plugin_manifest.name); + return Ok(()); + } + let temp_dir = tempdir()?; + let plugin_file_name = + PluginInstaller::download_plugin(&plugin_manifest.name, &temp_dir, &target_url).await?; + self.verify_checksum(&plugin_file_name, &plugin_package.sha256)?; + + self.untar_plugin(&plugin_file_name, &plugin_manifest.name)?; + // Save manifest to installed plugins directory + self.add_to_manifest_dir(&plugin_manifest)?; + log::info!("Plugin installed successfully"); + Ok(()) + } + + fn untar_plugin(&self, plugin_file_name: &PathBuf, plugin_name: &str) -> Result<()> { + // Get handle to file + let tar_gz = File::open(&plugin_file_name)?; + // Unzip file + let tar = GzDecoder::new(tar_gz); + // Get plugin from tarball + let mut archive = Archive::new(tar); + // TODO: this is unix only. Look into whether permissions are preserved + archive.set_preserve_permissions(true); + // Create subdirectory in plugins directory for this plugin + let plugin_sub_dir = self.plugins_dir.join(plugin_name); + fs::create_dir_all(&plugin_sub_dir)?; + archive.unpack(&plugin_sub_dir)?; + Ok(()) + } + + async fn download_plugin(name: &str, temp_dir: &TempDir, target_url: &str) -> Result { + log::info!( + "Trying to get tar file for plugin {} from {}", + name, + target_url + ); + let plugin_bin = reqwest::get(target_url).await?; + let mut content = Cursor::new(plugin_bin.bytes().await?); + let dir = temp_dir.path(); + let mut plugin_file = dir.join(name); + plugin_file.set_extension("tar.gz"); + let mut temp_file = File::create(&plugin_file)?; + copy(&mut content, &mut temp_file)?; + Ok(plugin_file) + } + + // Validate checksum of downloaded content with checksum from Index + fn verify_checksum(&self, plugin_file: &PathBuf, checksum: &str) -> Result<()> { + let binary_sha256 = file_digest_string(plugin_file).expect("failed to get sha for parcel"); + let verification_sha256 = checksum; + if binary_sha256 == verification_sha256 { + println!("Package verified successfully"); + Ok(()) + } else { + Err(anyhow!("Could not validate Checksum")) + } + } + + fn add_to_manifest_dir(&self, plugin: &PluginManifest) -> Result<()> { + let manifests_dir = self.plugins_dir.join(PLUGIN_MANIFESTS_DIRECTORY_NAME); + fs::create_dir_all(&manifests_dir)?; + serde_json::to_writer( + &File::create(manifests_dir.join(get_manifest_file_name(&plugin.name)))?, + plugin, + )?; + Ok(()) + } +} + +fn get_manifest_file_name(plugin_name: &str) -> String { + format!("{}.json", plugin_name) +} + +fn file_digest_string(path: impl AsRef) -> Result { + use sha2::{Digest, Sha256}; + let mut file = std::fs::File::open(&path)?; + let mut sha = Sha256::new(); + std::io::copy(&mut file, &mut sha)?; + let digest_value = sha.finalize(); + let digest_string = format!("{:x}", digest_value); + Ok(digest_string) +} diff --git a/crates/plugins/src/lib.rs b/crates/plugins/src/lib.rs new file mode 100644 index 0000000000..f0ec891004 --- /dev/null +++ b/crates/plugins/src/lib.rs @@ -0,0 +1,5 @@ +mod plugin_manifest; +// TODO: just export PluginInstaller +mod git; +pub mod install; +mod prompt; diff --git a/crates/plugins/src/plugin_manifest.rs b/crates/plugins/src/plugin_manifest.rs new file mode 100644 index 0000000000..b589144447 --- /dev/null +++ b/crates/plugins/src/plugin_manifest.rs @@ -0,0 +1,136 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PluginManifest { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + homepage: Option, + version: String, + spin_compatibility: String, + pub license: String, + pub packages: Vec, +} + +#[derive(Serialize, Debug, Deserialize, PartialEq)] +pub(crate) struct PluginPackage { + pub os: Os, + pub arch: Architecture, + pub url: String, + pub sha256: String, +} + +#[derive(Serialize, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Os { + Linux, + Osx, + Windows, +} + +#[derive(Serialize, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum Architecture { + Amd64, + Aarch64, +} + +// TODO: create licenses enum + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_plugin_json() { + let name = "test"; + let description = "Some description."; + let homepage = "www.example.com"; + let version = "1.0"; + let license = "Mit"; + let plugin_json = r#" + { + "name": "test", + "description": "Some description.", + "homepage": "www.example.com", + "version": "1.0", + "spinCompatibility": "=0.4", + "license": "Mit", + "packages": [ + { + "os": "linux", + "arch": "amd64", + "url": "www.example.com/releases/1.0/binary.tgz", + "sha256": "c474f00b12345e38acae2d19b2a707a4fhdjdfdd22875efeefdf052ce19c90b" + }, + { + "os": "windows", + "arch": "amd64", + "url": "www.example.com/releases/1.0/binary.tgz", + "sha256": "eee4f00b12345e38acae2d19b2a707a4fhdjdfdd22875efeefdf052ce19c90b" + }, + { + "os": "osx", + "arch": "aarch64", + "url": "www.example.com/releases/1.0/binary.tgz", + "sha256": "eeegf00b12345e38acae2d19b2a707a4fhdjdfdd22875efeefdf052ce19c90b" + } + ] + }"#; + + let deserialized_plugin: PluginManifest = serde_json::from_str(plugin_json).unwrap(); + assert_eq!(deserialized_plugin.name, name.to_owned()); + assert_eq!( + deserialized_plugin.description, + Some(description.to_owned()) + ); + assert_eq!(deserialized_plugin.homepage, Some(homepage.to_owned())); + assert_eq!(deserialized_plugin.version, version.to_owned()); + assert_eq!(deserialized_plugin.license, license.to_owned()); + assert_eq!(deserialized_plugin.packages.len(), 3); + } + + #[test] + fn test_plugin_json_empty_options() { + let name = "test"; + let version = "1.0"; + let license = "Mit"; + let plugin_json = r#" + { + "name": "test", + "version": "1.0", + "spinCompatibility": "=0.4", + "license": "Mit", + "packages": [ + { + "os": "linux", + "arch": "amd64", + "url": "www.example.com/releases/1.0/binary.tgz", + "sha256": "c474f00b12345e38acae2d19b2a707a4fhdjdfdd22875efeefdf052ce19c90b" + }, + { + "os": "windows", + "arch": "amd64", + "url": "www.example.com/releases/1.0/binary.tgz", + "sha256": "eee4f00b12345e38acae2d19b2a707a4fhdjdfdd22875efeefdf052ce19c90b" + }, + { + "os": "osx", + "arch": "aarch64", + "url": "www.example.com/releases/1.0/binary.tgz", + "sha256": "eeegf00b12345e38acae2d19b2a707a4fhdjdfdd22875efeefdf052ce19c90b" + } + ] + }"#; + + let deserialized_plugin: PluginManifest = serde_json::from_str(plugin_json).unwrap(); + assert_eq!(deserialized_plugin.name, name.to_owned()); + assert_eq!(deserialized_plugin.description, None); + assert_eq!(deserialized_plugin.homepage, None); + assert_eq!(deserialized_plugin.version, version.to_owned()); + assert_eq!(deserialized_plugin.license, license.to_owned()); + assert_eq!(deserialized_plugin.packages.len(), 3); + } +} diff --git a/crates/plugins/src/prompt.rs b/crates/plugins/src/prompt.rs new file mode 100644 index 0000000000..820442e225 --- /dev/null +++ b/crates/plugins/src/prompt.rs @@ -0,0 +1,61 @@ +/// Prompts user as to whether they trust the source of the plugin and +/// want to proceed with installation +use anyhow::Result; +use std::io; + +pub(crate) struct Prompter { + plugin_name: String, + plugin_license: String, + source_url: String, +} + +impl Prompter { + pub fn new(plugin_name: &str, plugin_license: &str, source_url: &str) -> Result { + Ok(Self { + plugin_name: plugin_name.to_string(), + plugin_license: plugin_license.to_string(), + source_url: source_url.to_string(), + }) + } + fn print_prompt(&self) { + println!( + "Installing plugin {} with license {} from {}\n", + self.plugin_name, self.plugin_license, self.source_url + ); + println!("Are you sure you want to proceed? (y/N)"); + } + + fn are_you_sure(&self) -> Result { + let mut resp = String::new(); + io::stdin().read_line(&mut resp)?; + Ok(self.parse_response(&mut resp)) + } + fn parse_response(&self, resp: &mut str) -> bool { + let resp = resp.trim().to_lowercase(); + resp.eq("yes") || resp.eq("y") + // TODO: consider checking for invalid response + } + + // Returns whether or not the user would like to proceed with the installation + pub(crate) fn run(&self) -> Result { + self.print_prompt(); + self.are_you_sure() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_parse_response() { + let p = Prompter::new( + "best-plugin", + "MIT", + "www.example.com/releases/example-1.0.tar.gz", + ) + .unwrap(); + let mut resp = String::from("\n\t yes "); + assert!(p.parse_response(&mut resp)); + } +} diff --git a/src/bin/spin.rs b/src/bin/spin.rs index cc67d46595..2890ef2bf7 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -2,7 +2,8 @@ use anyhow::Error; use clap::{Parser, Subcommand}; use lazy_static::lazy_static; use spin_cli::commands::{ - bindle::BindleCommands, build::BuildCommand, deploy::DeployCommand, new::NewCommand, + bindle::BindleCommands, build::BuildCommand, deploy::DeployCommand, + external::execute_external_subcommand, new::NewCommand, plugins::PluginCommands, templates::TemplateCommands, up::UpCommand, }; use spin_http_engine::HttpTrigger; @@ -16,7 +17,6 @@ async fn main() -> Result<(), Error> { .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .with_ansi(atty::is(atty::Stream::Stderr)) .init(); - SpinApp::parse().run().await } @@ -44,8 +44,12 @@ enum SpinApp { Bindle(BindleCommands), Deploy(DeployCommand), Build(BuildCommand), + #[clap(subcommand)] + Plugin(PluginCommands), #[clap(subcommand, hide = true)] Trigger(TriggerCommands), + #[clap(external_subcommand)] + External(Vec), } #[derive(Subcommand)] @@ -66,6 +70,8 @@ impl SpinApp { Self::Build(cmd) => cmd.run().await, Self::Trigger(TriggerCommands::Http(cmd)) => cmd.run().await, Self::Trigger(TriggerCommands::Redis(cmd)) => cmd.run().await, + Self::Plugin(cmd) => cmd.run().await, + Self::External(cmd) => execute_external_subcommand(cmd).await, } } } diff --git a/src/commands.rs b/src/commands.rs index f8676fa378..9badc7abe0 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -6,8 +6,12 @@ pub mod bindle; pub mod build; /// Command for deploying a Spin app to Hippo pub mod deploy; +/// commands for external subcommands +pub mod external; /// Command for creating a new application. pub mod new; +/// Command for adding a plugin to Spin +pub mod plugins; /// Commands for working with templates. pub mod templates; /// Commands for starting the runtime. diff --git a/src/commands/external.rs b/src/commands/external.rs new file mode 100644 index 0000000000..a34bd6145d --- /dev/null +++ b/src/commands/external.rs @@ -0,0 +1,26 @@ +use anyhow::Result; +use tokio::process::Command; +use tracing::log; + +use crate::commands::plugins::get_spin_plugins_directory; + +// TODO: Add capability to distinguish between standalone binaries and pluigns +// TODO: Should this be a struct to maintain consistency across subcommands? + +pub async fn execute_external_subcommand(args: Vec) -> Result<()> { + // TODO: What environmental variables should be passed. + + let path = get_spin_plugins_directory()? + .join(args.first().unwrap()) + .join(args.first().unwrap()); + let mut command = Command::new(path); + if args.len() > 1 { + command.args(&args[1..]); + } + log::info!("Executing command {:?}", command); + // Allow user to interact with stdio/stdout of child process + let _ = command.status().await?; + // TODO: handle the status + + Ok(()) +} diff --git a/src/commands/plugins.rs b/src/commands/plugins.rs new file mode 100644 index 0000000000..d4af879355 --- /dev/null +++ b/src/commands/plugins.rs @@ -0,0 +1,124 @@ +use anyhow::{anyhow, Result}; +use clap::{Parser, Subcommand}; +use semver::Version; +use spin_plugins::install::{ManifestLocation, PluginInfo, PluginInstaller}; +use std::path::PathBuf; +use url::Url; + +const SPIN_PLUGINS_REPO: &str = "https://github.com/fermyon/spin-plugins/"; + +/// Install/uninstall plugins +#[derive(Subcommand, Debug)] +pub enum PluginCommands { + /// Install plugin from the Spin plugin repository. + /// + /// The binary or .wasm file of the plugin is copied to the local Spin plugins directory + /// TODO: consider the ability to install multiple plugins + Install(Install), + + /// Remove a plugin from your installation. + Uninstall(Uninstall), + // TODO: consider Search command + + // TODO: consider List command +} + +impl PluginCommands { + pub async fn run(self) -> Result<()> { + match self { + PluginCommands::Install(cmd) => cmd.run().await, + PluginCommands::Uninstall(cmd) => cmd.run().await, + } + } +} + +/// Install plugins from remote source +#[derive(Parser, Debug)] +pub struct Install { + /// Name of Spin plugin. + #[clap( + name = "PLUGIN_NAME", + conflicts_with = "REMOTE_PLUGIN_MANIFEST", + conflicts_with = "LOCAL_PLUGIN_MANIFEST", + required_unless_present_any = ["REMOTE_PLUGIN_MANIFEST", "LOCAL_PLUGIN_MANIFEST"], + )] + pub name: Option, + /// source of local manifest file + #[clap( + name = "LOCAL_PLUGIN_MANIFEST", + short = 'f', + long = "file", + conflicts_with = "REMOTE_PLUGIN_MANIFEST", + conflicts_with = "PLUGIN_NAME" + )] + pub local_manifest_src: Option, + /// source of remote manifest file + #[clap( + name = "REMOTE_PLUGIN_MANIFEST", + short = 'u', + long = "url", + conflicts_with = "LOCAL_PLUGIN_MANIFEST", + conflicts_with = "PLUGIN_NAME" + )] + pub remote_manifest_src: Option, + /// skips prompt to accept the installation of the plugin. + #[clap(short = 'y', long = "yes", takes_value = false)] + pub yes_to_all: bool, + /// specify particular version of plugin to install from centralized repository + #[clap( + long = "version", + short = 'v', + conflicts_with = "REMOTE_PLUGIN_MANIFEST", + conflicts_with = "LOCAL_PLUGIN_MANIFEST", + requires("PLUGIN_NAME") + )] + /// Specify a particular version of the plugin to be installed from the Centralized Repository + pub version: Option, +} + +impl Install { + pub async fn run(self) -> Result<()> { + println!("Attempting to install plugin: {:?}", self.name); + let manifest_location = match (self.local_manifest_src, self.remote_manifest_src, self.name) { + // TODO: move all this parsing into clap to catch input errors. + (Some(path), None, None) => ManifestLocation::Local(path), + (None, Some(url), None) => ManifestLocation::Remote(url), + (None, None, Some(name)) => ManifestLocation::PluginsRepository(PluginInfo::new(name, Url::parse(SPIN_PLUGINS_REPO)?)), + _ => return Err(anyhow::anyhow!("Must provide plugin name for plugin look up xor remote xor local path to plugin manifest")), + }; + PluginInstaller::new( + manifest_location, + get_spin_plugins_directory()?, + self.yes_to_all, + ) + .install() + .await?; + Ok(()) + } +} + +/// Remove the specified plugin +#[derive(Parser, Debug)] +pub struct Uninstall { + /// Name of Spin plugin. + pub name: String, + // TODO: think about how to handle breaking changes + // #[structopt(long = "update")] + // pub update: bool, +} + +impl Uninstall { + pub async fn run(self) -> Result<()> { + println!("The plugin {:?} will be removed", self.name); + Ok(()) + } +} + +/// Gets the path to where Spin plugin are (to be) installed +pub fn get_spin_plugins_directory() -> anyhow::Result { + let data_dir = dirs::data_local_dir() + .or_else(|| dirs::home_dir().map(|p| p.join(".spin"))) + .ok_or_else(|| anyhow!("Unable to get local data directory or home directory"))?; + let plugins_dir = data_dir.join("spin").join("plugins"); + Ok(plugins_dir) +} From 146d5c402c67ec97cbc10edb1e9dc1614e7828c4 Mon Sep 17 00:00:00 2001 From: karthik Ganeshram Date: Sun, 28 Aug 2022 17:44:26 -0700 Subject: [PATCH 003/163] feat: support uninstall sub-function for spin plugins also add Spin compatibility checks, version checks, upgrading via installation. Signed-off-by: karthik Ganeshram Co-authored-by: Kate Goldenring --- Cargo.lock | 1 + crates/plugins/Cargo.toml | 3 +- crates/plugins/src/git.rs | 4 +- crates/plugins/src/install.rs | 55 ++++++++++++++++++----- crates/plugins/src/lib.rs | 21 ++++++++- crates/plugins/src/plugin_manifest.rs | 4 +- crates/plugins/src/uninstall.rs | 39 ++++++++++++++++ crates/plugins/src/version_check.rs | 65 +++++++++++++++++++++++++++ src/commands/external.rs | 13 +++--- src/commands/plugins.rs | 11 +++-- 10 files changed, 187 insertions(+), 29 deletions(-) create mode 100644 crates/plugins/src/uninstall.rs create mode 100644 crates/plugins/src/version_check.rs diff --git a/Cargo.lock b/Cargo.lock index d4fed602f2..378aec17d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3470,6 +3470,7 @@ dependencies = [ "flate2", "log", "reqwest", + "semver 1.0.6", "serde", "serde_json", "sha2 0.10.3", diff --git a/crates/plugins/Cargo.toml b/crates/plugins/Cargo.toml index c25f3035b7..6bd89d6ff5 100644 --- a/crates/plugins/Cargo.toml +++ b/crates/plugins/Cargo.toml @@ -11,9 +11,10 @@ bytes = "1.1" flate2 = "1.0" log = { version = "0.4", default-features = false } reqwest = { version = "0.11", features = ["json"] } -sha2 = "0.10.2" +semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.10.2" tar = "0.4.38" tempfile = "3.3.0" tokio = { version = "1.10", features = [ "fs", "process", "rt", "macros" ] } diff --git a/crates/plugins/src/git.rs b/crates/plugins/src/git.rs index 6a13182125..4e42f3e1e5 100644 --- a/crates/plugins/src/git.rs +++ b/crates/plugins/src/git.rs @@ -28,7 +28,7 @@ impl GitSource { let mut git = Command::new("git"); git.args([ "clone", - &self.source_url.to_string(), + self.source_url.as_ref(), "--branch", &self.branch, "--single-branch", @@ -55,7 +55,7 @@ impl GitSource { let pull_result = git.output().await?; match pull_result.status.success() { true => { - println!("Updated Repository Successfully"); + println!("Updated repository successfully"); Ok(()) } false => Err(anyhow!( diff --git a/crates/plugins/src/install.rs b/crates/plugins/src/install.rs index 3fd60584e9..2127fc2e4a 100644 --- a/crates/plugins/src/install.rs +++ b/crates/plugins/src/install.rs @@ -1,8 +1,15 @@ +use crate::{ + get_manifest_file_name, get_manifest_file_name_version, + version_check::{assert_supported_version, get_plugin_manifest}, + PLUGIN_MANIFESTS_DIRECTORY_NAME, +}; + use super::git::GitSource; use super::plugin_manifest::{Os, PluginManifest}; use super::prompt::Prompter; use anyhow::{anyhow, Result}; use flate2::read::GzDecoder; +use semver::Version; use std::{ fs::{self, File}, io::{copy, Cursor}, @@ -13,7 +20,6 @@ use tempfile::{tempdir, TempDir}; use url::Url; /// Name of the subdirectory that contains the installed plugin JSON manifests -const PLUGIN_MANIFESTS_DIRECTORY_NAME: &str = "manifests"; const PLUGINS_REPO_LOCAL_DIRECTORY: &str = ".spin-plugins"; const PLUGINS_REPO_MANIFESTS_DIRECTORY: &str = "manifests"; @@ -26,11 +32,15 @@ pub enum ManifestLocation { pub struct PluginInfo { name: String, repo_url: Url, - // version + version: Option, } impl PluginInfo { - pub fn new(name: String, repo_url: Url) -> Self { - Self { name, repo_url } + pub fn new(name: String, repo_url: Url, version: Option) -> Self { + Self { + name, + repo_url, + version, + } } } @@ -38,6 +48,7 @@ pub struct PluginInstaller { manifest_location: ManifestLocation, plugins_dir: PathBuf, yes_to_all: bool, + spin_version: String, } impl PluginInstaller { @@ -45,11 +56,13 @@ impl PluginInstaller { manifest_location: ManifestLocation, plugins_dir: PathBuf, yes_to_all: bool, + spin_version: &str, ) -> Self { Self { manifest_location, plugins_dir, yes_to_all, + spin_version: spin_version.to_string(), } } @@ -88,10 +101,8 @@ impl PluginInstaller { .exists() { git_source.clone().await?; - // self.get_latest_plugin_repo(&info.repo_url)?; } else { git_source.pull().await?; - // self.update_plugins_repository()?; } let file = File::open( &self @@ -99,12 +110,36 @@ impl PluginInstaller { .join(PLUGINS_REPO_LOCAL_DIRECTORY) .join(PLUGINS_REPO_MANIFESTS_DIRECTORY) .join(&info.name) - .join(get_manifest_file_name(&info.name)), - )?; + .join(get_manifest_file_name_version(&info.name, &info.version)), + ) + .map_err(|_| { + anyhow!( + "Could not find plugin [{} {:?}] in centralized repository", + info.name, + info.version + .as_ref() + .map(|s| s.to_string()) + .unwrap_or_else(|| String::from("latest")) + ) + })?; serde_json::from_reader(file)? } }; + // Return early if plugin is already installed with latest version + if let Ok(installed) = get_plugin_manifest(&plugin_manifest.name, &self.plugins_dir) { + if installed.version >= plugin_manifest.version { + return Err(anyhow!( + "plugin {} already installed with version {} but attempting to install same or older version ({})", + installed.name, + installed.version, + plugin_manifest.version, + )); + } + } + + assert_supported_version(&self.spin_version, &plugin_manifest.spin_compatibility)?; + let os: Os = if cfg!(target_os = "windows") { Os::Windows } else if cfg!(target_os = "linux") { @@ -198,10 +233,6 @@ impl PluginInstaller { } } -fn get_manifest_file_name(plugin_name: &str) -> String { - format!("{}.json", plugin_name) -} - fn file_digest_string(path: impl AsRef) -> Result { use sha2::{Digest, Sha256}; let mut file = std::fs::File::open(&path)?; diff --git a/crates/plugins/src/lib.rs b/crates/plugins/src/lib.rs index f0ec891004..d564618577 100644 --- a/crates/plugins/src/lib.rs +++ b/crates/plugins/src/lib.rs @@ -1,5 +1,22 @@ -mod plugin_manifest; -// TODO: just export PluginInstaller mod git; +// TODO: just export PluginInstaller pub mod install; +mod plugin_manifest; mod prompt; +pub mod uninstall; +pub mod version_check; + +/// Directory where the manifests of installed plugins are stored. +const PLUGIN_MANIFESTS_DIRECTORY_NAME: &str = "manifests"; + +fn get_manifest_file_name(plugin_name: &str) -> String { + format!("{}.json", plugin_name) +} + +// Given a name and option version, outputs expected file name for the plugin. +fn get_manifest_file_name_version(plugin_name: &str, version: &Option) -> String { + match version { + Some(v) => format!("{}@{}.json", plugin_name, v), + None => get_manifest_file_name(plugin_name), + } +} diff --git a/crates/plugins/src/plugin_manifest.rs b/crates/plugins/src/plugin_manifest.rs index b589144447..8a0e4b4e2a 100644 --- a/crates/plugins/src/plugin_manifest.rs +++ b/crates/plugins/src/plugin_manifest.rs @@ -8,8 +8,8 @@ pub(crate) struct PluginManifest { description: Option, #[serde(default, skip_serializing_if = "Option::is_none")] homepage: Option, - version: String, - spin_compatibility: String, + pub version: String, + pub spin_compatibility: String, pub license: String, pub packages: Vec, } diff --git a/crates/plugins/src/uninstall.rs b/crates/plugins/src/uninstall.rs new file mode 100644 index 0000000000..eef879f499 --- /dev/null +++ b/crates/plugins/src/uninstall.rs @@ -0,0 +1,39 @@ +use crate::{get_manifest_file_name, PLUGIN_MANIFESTS_DIRECTORY_NAME}; +use anyhow::Result; +use std::{fs, path::PathBuf}; +pub struct PluginUninstaller { + plugin_name: String, + plugins_dir: PathBuf, +} + +impl PluginUninstaller { + pub fn new(plugin_name: &str, plugins_dir: PathBuf) -> Self { + Self { + plugin_name: plugin_name.to_owned(), + plugins_dir, + } + } + pub fn run(&self) -> Result<()> { + // Check if plugin is installed + let manifest_file = self + .plugins_dir + .join(PLUGIN_MANIFESTS_DIRECTORY_NAME) + .join(get_manifest_file_name(&self.plugin_name)); + let plugin_exists = manifest_file.exists(); + match plugin_exists { + // Remove the manifest and the plugin installation directory + true => { + fs::remove_file(manifest_file)?; + fs::remove_dir_all(self.plugins_dir.join(&self.plugin_name))?; + println!("Uninstalled plugin {} successfully", &self.plugin_name); + } + false => { + println!( + "The following plugin \"{}\" does not exist, therefore cannot be uninstalled", + self.plugin_name + ); + } + } + Ok(()) + } +} diff --git a/crates/plugins/src/version_check.rs b/crates/plugins/src/version_check.rs new file mode 100644 index 0000000000..3aa9f75dbd --- /dev/null +++ b/crates/plugins/src/version_check.rs @@ -0,0 +1,65 @@ +use crate::{ + get_manifest_file_name, plugin_manifest::PluginManifest, PLUGIN_MANIFESTS_DIRECTORY_NAME, +}; +use anyhow::{anyhow, Result}; +use semver::{Version, VersionReq}; +use std::{fs::File, path::Path}; + +/// Checks whether the plugin supports the currently running version of Spin. +// TODO: check whether on main or canary (aka beyond the specified version). +pub fn assert_supported_version(spin_version: &str, supported: &str) -> Result<()> { + let supported = VersionReq::parse(supported).map_err(|e| { + anyhow!( + "could not parse manifest compatibility version {} as valid semver -- {:?}", + supported, + e + ) + })?; + let version = Version::parse(spin_version)?; + match supported.matches(&version) { + true => Ok(()), + false => Err(anyhow!( + "plugin is compatible with Spin {} but running Spin {}", + supported, + spin_version + )), + } +} + +pub(crate) fn get_plugin_manifest(plugin_name: &str, plugins_dir: &Path) -> Result { + let manifest_path = plugins_dir + .join(PLUGIN_MANIFESTS_DIRECTORY_NAME) + .join(get_manifest_file_name(plugin_name)); + log::info!("Reading plugin manifest from {:?}", manifest_path); + let manifest_file = File::open(manifest_path)?; + let manifest = serde_json::from_reader(manifest_file)?; + Ok(manifest) +} + +pub fn check_plugin_spin_compatibility( + plugin_name: &str, + spin_version: &str, + plugins_dir: &Path, +) -> Result<()> { + let manifest = get_plugin_manifest(plugin_name, plugins_dir)?; + assert_supported_version(spin_version, &manifest.spin_compatibility) +} + +#[cfg(test)] +mod version_tests { + use super::*; + #[test] + fn test_supported_version() { + let test_case = ">=1.2.3, <1.8.0"; + let input_output = [ + ("1.3.0", true), + ("1.2.3", true), + ("1.8.0", false), + ("1.9.0", false), + ("1.2.0", false), + ]; + input_output + .into_iter() + .for_each(|(i, o)| assert_eq!(assert_supported_version(i, test_case).is_err(), !o)); + } +} diff --git a/src/commands/external.rs b/src/commands/external.rs index a34bd6145d..c42a4f9d3a 100644 --- a/src/commands/external.rs +++ b/src/commands/external.rs @@ -1,18 +1,19 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; +use spin_plugins::version_check::check_plugin_spin_compatibility; use tokio::process::Command; use tracing::log; use crate::commands::plugins::get_spin_plugins_directory; -// TODO: Add capability to distinguish between standalone binaries and pluigns +// TODO: Add capability to distinguish between standalone binaries and plugins // TODO: Should this be a struct to maintain consistency across subcommands? pub async fn execute_external_subcommand(args: Vec) -> Result<()> { // TODO: What environmental variables should be passed. - - let path = get_spin_plugins_directory()? - .join(args.first().unwrap()) - .join(args.first().unwrap()); + let plugin_name = args.first().ok_or_else(|| anyhow!("Expected subcommand"))?; + let plugins_dir = get_spin_plugins_directory()?; + check_plugin_spin_compatibility(plugin_name, env!("VERGEN_BUILD_SEMVER"), &plugins_dir)?; + let path = plugins_dir.join(plugin_name).join(plugin_name); let mut command = Command::new(path); if args.len() > 1 { command.args(&args[1..]); diff --git a/src/commands/plugins.rs b/src/commands/plugins.rs index d4af879355..bf5779eb4b 100644 --- a/src/commands/plugins.rs +++ b/src/commands/plugins.rs @@ -1,7 +1,10 @@ use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use semver::Version; -use spin_plugins::install::{ManifestLocation, PluginInfo, PluginInstaller}; +use spin_plugins::{ + install::{ManifestLocation, PluginInfo, PluginInstaller}, + uninstall::PluginUninstaller, +}; use std::path::PathBuf; use url::Url; @@ -78,18 +81,18 @@ pub struct Install { impl Install { pub async fn run(self) -> Result<()> { - println!("Attempting to install plugin: {:?}", self.name); let manifest_location = match (self.local_manifest_src, self.remote_manifest_src, self.name) { // TODO: move all this parsing into clap to catch input errors. (Some(path), None, None) => ManifestLocation::Local(path), (None, Some(url), None) => ManifestLocation::Remote(url), - (None, None, Some(name)) => ManifestLocation::PluginsRepository(PluginInfo::new(name, Url::parse(SPIN_PLUGINS_REPO)?)), + (None, None, Some(name)) => ManifestLocation::PluginsRepository(PluginInfo::new(name, Url::parse(SPIN_PLUGINS_REPO)?, self.version)), _ => return Err(anyhow::anyhow!("Must provide plugin name for plugin look up xor remote xor local path to plugin manifest")), }; PluginInstaller::new( manifest_location, get_spin_plugins_directory()?, self.yes_to_all, + env!("VERGEN_BUILD_SEMVER"), ) .install() .await?; @@ -109,7 +112,7 @@ pub struct Uninstall { impl Uninstall { pub async fn run(self) -> Result<()> { - println!("The plugin {:?} will be removed", self.name); + PluginUninstaller::new(&self.name, get_spin_plugins_directory()?).run()?; Ok(()) } } From 53bcc3cf846b042fa3b6b116bdd24ae20c436a83 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Mon, 29 Aug 2022 23:10:40 -0700 Subject: [PATCH 004/163] feat: add support for upgrade sub-command for Spin plugins Signed-off-by: Kate Goldenring --- crates/plugins/src/install.rs | 31 +++++--- crates/plugins/src/lib.rs | 2 +- src/commands/plugins.rs | 139 +++++++++++++++++++++++++++++++++- 3 files changed, 158 insertions(+), 14 deletions(-) diff --git a/crates/plugins/src/install.rs b/crates/plugins/src/install.rs index 2127fc2e4a..06390fbfec 100644 --- a/crates/plugins/src/install.rs +++ b/crates/plugins/src/install.rs @@ -22,6 +22,8 @@ use url::Url; /// Name of the subdirectory that contains the installed plugin JSON manifests const PLUGINS_REPO_LOCAL_DIRECTORY: &str = ".spin-plugins"; const PLUGINS_REPO_MANIFESTS_DIRECTORY: &str = "manifests"; +/// Anticipated url scheme prefix in manifest to indicate local path to plugin binary +const URL_FILE_SCHEME: &str = "file"; pub enum ManifestLocation { Local(PathBuf), @@ -35,9 +37,9 @@ pub struct PluginInfo { version: Option, } impl PluginInfo { - pub fn new(name: String, repo_url: Url, version: Option) -> Self { + pub fn new(name: &str, repo_url: Url, version: Option) -> Self { Self { - name, + name: name.to_string(), repo_url, version, } @@ -126,9 +128,9 @@ impl PluginInstaller { } }; - // Return early if plugin is already installed with latest version + // Disallow downgrades and reinstalling identical plugins if let Ok(installed) = get_plugin_manifest(&plugin_manifest.name, &self.plugins_dir) { - if installed.version >= plugin_manifest.version { + if installed.version > plugin_manifest.version || installed == plugin_manifest { return Err(anyhow!( "plugin {} already installed with version {} but attempting to install same or older version ({})", installed.name, @@ -155,26 +157,30 @@ impl PluginInstaller { .iter() .find(|p| p.os == os) .ok_or_else(|| anyhow!("This plugin does not support this OS"))?; - let target_url = plugin_package.url.to_owned(); + let target = plugin_package.url.to_owned(); // Ask for user confirmation if not overridden with CLI option if !self.yes_to_all - && !Prompter::new(&plugin_manifest.name, &plugin_manifest.license, &target_url)? - .run()? + && !Prompter::new(&plugin_manifest.name, &plugin_manifest.license, &target)?.run()? { // User has requested to not install package, returning early println!("Plugin {} will not be installed", plugin_manifest.name); return Ok(()); } + let target_url = Url::parse(&target)?; let temp_dir = tempdir()?; - let plugin_file_name = - PluginInstaller::download_plugin(&plugin_manifest.name, &temp_dir, &target_url).await?; - self.verify_checksum(&plugin_file_name, &plugin_package.sha256)?; + let plugin_tarball_path = match target_url.scheme() { + URL_FILE_SCHEME => PathBuf::from(target_url.path()), + _ => { + PluginInstaller::download_plugin(&plugin_manifest.name, &temp_dir, &target).await? + } + }; + self.verify_checksum(&plugin_tarball_path, &plugin_package.sha256)?; - self.untar_plugin(&plugin_file_name, &plugin_manifest.name)?; + self.untar_plugin(&plugin_tarball_path, &plugin_manifest.name)?; // Save manifest to installed plugins directory self.add_to_manifest_dir(&plugin_manifest)?; - log::info!("Plugin installed successfully"); + log::info!("Plugin [{}] installed successfully", plugin_manifest.name); Ok(()) } @@ -189,6 +195,7 @@ impl PluginInstaller { archive.set_preserve_permissions(true); // Create subdirectory in plugins directory for this plugin let plugin_sub_dir = self.plugins_dir.join(plugin_name); + fs::remove_dir_all(&plugin_sub_dir).ok(); fs::create_dir_all(&plugin_sub_dir)?; archive.unpack(&plugin_sub_dir)?; Ok(()) diff --git a/crates/plugins/src/lib.rs b/crates/plugins/src/lib.rs index d564618577..2a396a1cbf 100644 --- a/crates/plugins/src/lib.rs +++ b/crates/plugins/src/lib.rs @@ -7,7 +7,7 @@ pub mod uninstall; pub mod version_check; /// Directory where the manifests of installed plugins are stored. -const PLUGIN_MANIFESTS_DIRECTORY_NAME: &str = "manifests"; +pub const PLUGIN_MANIFESTS_DIRECTORY_NAME: &str = "manifests"; fn get_manifest_file_name(plugin_name: &str) -> String { format!("{}.json", plugin_name) diff --git a/src/commands/plugins.rs b/src/commands/plugins.rs index bf5779eb4b..e93b42a5b1 100644 --- a/src/commands/plugins.rs +++ b/src/commands/plugins.rs @@ -4,8 +4,10 @@ use semver::Version; use spin_plugins::{ install::{ManifestLocation, PluginInfo, PluginInstaller}, uninstall::PluginUninstaller, + PLUGIN_MANIFESTS_DIRECTORY_NAME, }; use std::path::PathBuf; +use tracing::log; use url::Url; const SPIN_PLUGINS_REPO: &str = "https://github.com/fermyon/spin-plugins/"; @@ -21,6 +23,9 @@ pub enum PluginCommands { /// Remove a plugin from your installation. Uninstall(Uninstall), + + /// Upgrade one or all plugins + Upgrade(Upgrade), // TODO: consider Search command // TODO: consider List command @@ -31,6 +36,7 @@ impl PluginCommands { match self { PluginCommands::Install(cmd) => cmd.run().await, PluginCommands::Uninstall(cmd) => cmd.run().await, + PluginCommands::Upgrade(cmd) => cmd.run().await, } } } @@ -85,7 +91,7 @@ impl Install { // TODO: move all this parsing into clap to catch input errors. (Some(path), None, None) => ManifestLocation::Local(path), (None, Some(url), None) => ManifestLocation::Remote(url), - (None, None, Some(name)) => ManifestLocation::PluginsRepository(PluginInfo::new(name, Url::parse(SPIN_PLUGINS_REPO)?, self.version)), + (None, None, Some(name)) => ManifestLocation::PluginsRepository(PluginInfo::new(&name, Url::parse(SPIN_PLUGINS_REPO)?, self.version)), _ => return Err(anyhow::anyhow!("Must provide plugin name for plugin look up xor remote xor local path to plugin manifest")), }; PluginInstaller::new( @@ -117,6 +123,137 @@ impl Uninstall { } } +#[derive(Parser, Debug)] +pub struct Upgrade { + /// Name of Spin plugin to upgrade. + #[clap( + name = "PLUGIN_NAME", + conflicts_with = "ALL", + required_unless_present_any = ["ALL"], + )] + pub name: Option, + /// Upgrade all plugins + #[clap( + short = 'a', + long = "all", + name = "ALL", + conflicts_with = "PLUGIN_NAME", + takes_value = false + )] + pub all: bool, + /// Source of local manifest file + #[clap( + name = "LOCAL_PLUGIN_MANIFEST", + short = 'f', + long = "file", + conflicts_with = "REMOTE_PLUGIN_MANIFEST", + conflicts_with = "PLUGIN_NAME" + )] + pub local_manifest_src: Option, + /// Source of remote manifest file + #[clap( + name = "REMOTE_PLUGIN_MANIFEST", + short = 'u', + long = "url", + conflicts_with = "LOCAL_PLUGIN_MANIFEST", + conflicts_with = "PLUGIN_NAME" + )] + pub remote_manifest_src: Option, + /// skips prompt to accept the installation of the plugin. + #[clap(short = 'y', long = "yes", takes_value = false)] + pub yes_to_all: bool, + #[clap( + long = "version", + short = 'v', + conflicts_with = "REMOTE_PLUGIN_MANIFEST", + conflicts_with = "LOCAL_PLUGIN_MANIFEST", + requires("PLUGIN_NAME") + )] + /// Specify a particular version of the plugin to be installed from the centralized plugin repository + pub version: Option, + /// Allow downgrading a plugin's version + #[clap(short = 'd', long = "downgrade", takes_value = false)] + pub downgrade: bool, +} + +impl Upgrade { + pub async fn run(self) -> Result<()> { + let plugins_dir = get_spin_plugins_directory()?; + let spin_version = env!("VERGEN_BUILD_SEMVER"); + let manifest_dir = plugins_dir.join(PLUGIN_MANIFESTS_DIRECTORY_NAME); + + // Check if no plugins are currently installed + if !manifest_dir.exists() { + println!("No currently installed plugins to update."); + return Ok(()); + } + + if self.all { + // Install the latest of all currently installed plugins + for plugin in std::fs::read_dir(manifest_dir)? { + let path = plugin?.path(); + let name = path + .file_stem() + .ok_or_else(|| anyhow!("expected directory for plugin"))? + .to_str() + .ok_or_else(|| anyhow!("Cannot convert directory to String"))? + .to_string(); + if let Err(e) = PluginInstaller::new( + ManifestLocation::PluginsRepository(PluginInfo::new( + &name, + Url::parse(SPIN_PLUGINS_REPO)?, + None, + )), + plugins_dir.clone(), + self.yes_to_all, + spin_version, + ) + .install() + .await + { + // Ignore plugins that were not installed from the central plugins repository + if e.to_string().contains("Could not find plugin") { + log::info!( + "Could not update {} plugin as DNE in central repository", + name + ); + } else { + return Err(e); + } + } + } + } else { + let name = self + .name + .ok_or_else(|| anyhow!("plugin name is required for upgrades"))?; + // If downgrade is allowed, first uninstall the plugin + if self.downgrade { + PluginUninstaller::new(&name, plugins_dir.clone()).run()?; + } + let manifest_location = match (self.local_manifest_src, self.remote_manifest_src) { + // TODO: move all this parsing into clap to catch input errors. + (Some(path), None) => ManifestLocation::Local(path), + (None, Some(url)) => ManifestLocation::Remote(url), + _ => ManifestLocation::PluginsRepository(PluginInfo::new( + &name, + Url::parse(SPIN_PLUGINS_REPO)?, + self.version, + )), + }; + PluginInstaller::new( + manifest_location, + plugins_dir, + self.yes_to_all, + spin_version, + ) + .install() + .await?; + } + + Ok(()) + } +} + /// Gets the path to where Spin plugin are (to be) installed pub fn get_spin_plugins_directory() -> anyhow::Result { let data_dir = dirs::data_local_dir() From be5aa26f6b9a2a2d32f2e6dfe20a02a038a0f345 Mon Sep 17 00:00:00 2001 From: karthik Ganeshram Date: Tue, 30 Aug 2022 11:08:02 -0700 Subject: [PATCH 005/163] feat: add environment variables to be passed to plugin subcommand Signed-off-by: karthik Ganeshram --- crates/plugins/src/version_check.rs | 3 ++- src/commands/external.rs | 41 ++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/crates/plugins/src/version_check.rs b/crates/plugins/src/version_check.rs index 3aa9f75dbd..86fb4eaf43 100644 --- a/crates/plugins/src/version_check.rs +++ b/crates/plugins/src/version_check.rs @@ -31,7 +31,8 @@ pub(crate) fn get_plugin_manifest(plugin_name: &str, plugins_dir: &Path) -> Resu .join(PLUGIN_MANIFESTS_DIRECTORY_NAME) .join(get_manifest_file_name(plugin_name)); log::info!("Reading plugin manifest from {:?}", manifest_path); - let manifest_file = File::open(manifest_path)?; + let manifest_file = + File::open(manifest_path).map_err(|_| anyhow!("The plugin does not exist"))?; let manifest = serde_json::from_reader(manifest_file)?; Ok(manifest) } diff --git a/src/commands/external.rs b/src/commands/external.rs index c42a4f9d3a..bb59cb2b6f 100644 --- a/src/commands/external.rs +++ b/src/commands/external.rs @@ -1,27 +1,50 @@ use anyhow::{anyhow, Result}; use spin_plugins::version_check::check_plugin_spin_compatibility; +use std::{collections::HashMap, env, path::Path}; use tokio::process::Command; use tracing::log; use crate::commands::plugins::get_spin_plugins_directory; -// TODO: Add capability to distinguish between standalone binaries and plugins -// TODO: Should this be a struct to maintain consistency across subcommands? - pub async fn execute_external_subcommand(args: Vec) -> Result<()> { - // TODO: What environmental variables should be passed. let plugin_name = args.first().ok_or_else(|| anyhow!("Expected subcommand"))?; let plugins_dir = get_spin_plugins_directory()?; check_plugin_spin_compatibility(plugin_name, env!("VERGEN_BUILD_SEMVER"), &plugins_dir)?; - let path = plugins_dir.join(plugin_name).join(plugin_name); - let mut command = Command::new(path); + let path = plugins_dir.join(plugin_name); + let binary = path.join(plugin_name); + let mut command = Command::new(binary); if args.len() > 1 { command.args(&args[1..]); } + command.envs(&get_env_vars_map(&path)?); log::info!("Executing command {:?}", command); // Allow user to interact with stdio/stdout of child process - let _ = command.status().await?; - // TODO: handle the status - + let status = command.status().await?; + log::info!("Exiting process with {}", status); Ok(()) } + +fn get_env_vars_map(path: &Path) -> Result> { + let map: HashMap = vec![ + ( + "SPIN_VERSION".to_string(), + env!("VERGEN_BUILD_SEMVER").to_owned(), + ), + ( + "SPIN_BIN_PATH".to_string(), + env::current_exe()? + .to_str() + .ok_or_else(|| anyhow!("Could not convert binary path to string"))? + .to_string(), + ), + ( + "SPIN_PLUGIN_PATH".to_string(), + path.to_str() + .ok_or_else(|| anyhow!("Could not convert plugin path to string"))? + .to_string(), + ), + ] + .into_iter() + .collect(); + Ok(map) +} From ba362e76d6a2de86411b82988c5fbbdac2cf4b8b Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Fri, 2 Sep 2022 12:50:24 -0600 Subject: [PATCH 006/163] chore(*): update templates and examples to use the v0.5.0 sdk Signed-off-by: Vaughn Dice --- examples/http-rust-oubound-http/Cargo.lock | 2 +- examples/http-rust/Cargo.lock | 2 +- templates/Makefile | 2 +- templates/http-go/content/go.mod | 2 +- templates/http-go/content/go.sum | 4 ++-- templates/http-rust/content/Cargo.toml | 2 +- templates/redis-go/content/go.mod | 2 +- templates/redis-go/content/go.sum | 4 ++-- templates/redis-rust/content/Cargo.toml | 2 +- tests/http/simple-spin-rust/Cargo.lock | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/http-rust-oubound-http/Cargo.lock b/examples/http-rust-oubound-http/Cargo.lock index 06b733deb1..00ca2ba90a 100644 --- a/examples/http-rust-oubound-http/Cargo.lock +++ b/examples/http-rust-oubound-http/Cargo.lock @@ -154,7 +154,7 @@ dependencies = [ [[package]] name = "spin-sdk" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "bytes", diff --git a/examples/http-rust/Cargo.lock b/examples/http-rust/Cargo.lock index df527ec70c..bb7d8c8d9c 100644 --- a/examples/http-rust/Cargo.lock +++ b/examples/http-rust/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "spin-sdk" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "bytes", diff --git a/templates/Makefile b/templates/Makefile index afb8eac323..503be770ac 100644 --- a/templates/Makefile +++ b/templates/Makefile @@ -1,4 +1,4 @@ -SDK_VERSION ?= v0.3.0 +SDK_VERSION ?= v0.5.0 bump-versions: bump-go-versions bump-rust-versions diff --git a/templates/http-go/content/go.mod b/templates/http-go/content/go.mod index 76e8fbc7c0..188c97665a 100644 --- a/templates/http-go/content/go.mod +++ b/templates/http-go/content/go.mod @@ -2,4 +2,4 @@ module github.com/{{project-name | snake_case}} go 1.17 -require github.com/fermyon/spin/sdk/go v0.4.0 +require github.com/fermyon/spin/sdk/go v0.5.0 diff --git a/templates/http-go/content/go.sum b/templates/http-go/content/go.sum index 549fec5c79..3c8230cdba 100644 --- a/templates/http-go/content/go.sum +++ b/templates/http-go/content/go.sum @@ -1,2 +1,2 @@ -github.com/fermyon/spin/sdk/go v0.4.0 h1:aUcIK4IPx3gnHgQCIcrJ4sl+HZOITBoxEyp9Z18QqYA= -github.com/fermyon/spin/sdk/go v0.4.0/go.mod h1:ARV2oVtnUCykLM+xCBZq8MQrCZddzb3JbeBettYv1S0= +github.com/fermyon/spin/sdk/go v0.5.0 h1:rt0I8Vd18jtTXjKkCWsPy2DPBM7zXVbelhrXzNKskxg= +github.com/fermyon/spin/sdk/go v0.5.0/go.mod h1:ARV2oVtnUCykLM+xCBZq8MQrCZddzb3JbeBettYv1S0= diff --git a/templates/http-rust/content/Cargo.toml b/templates/http-rust/content/Cargo.toml index a0b143b137..e6f4a5efea 100644 --- a/templates/http-rust/content/Cargo.toml +++ b/templates/http-rust/content/Cargo.toml @@ -16,7 +16,7 @@ bytes = "1" # General-purpose crate with common HTTP types. http = "0.2" # The Spin SDK. -spin-sdk = { git = "https://github.com/fermyon/spin", tag = "v0.4.0" } +spin-sdk = { git = "https://github.com/fermyon/spin", tag = "v0.5.0" } # Crate that generates Rust Wasm bindings from a WebAssembly interface. wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } diff --git a/templates/redis-go/content/go.mod b/templates/redis-go/content/go.mod index 76e8fbc7c0..188c97665a 100644 --- a/templates/redis-go/content/go.mod +++ b/templates/redis-go/content/go.mod @@ -2,4 +2,4 @@ module github.com/{{project-name | snake_case}} go 1.17 -require github.com/fermyon/spin/sdk/go v0.4.0 +require github.com/fermyon/spin/sdk/go v0.5.0 diff --git a/templates/redis-go/content/go.sum b/templates/redis-go/content/go.sum index 549fec5c79..3c8230cdba 100644 --- a/templates/redis-go/content/go.sum +++ b/templates/redis-go/content/go.sum @@ -1,2 +1,2 @@ -github.com/fermyon/spin/sdk/go v0.4.0 h1:aUcIK4IPx3gnHgQCIcrJ4sl+HZOITBoxEyp9Z18QqYA= -github.com/fermyon/spin/sdk/go v0.4.0/go.mod h1:ARV2oVtnUCykLM+xCBZq8MQrCZddzb3JbeBettYv1S0= +github.com/fermyon/spin/sdk/go v0.5.0 h1:rt0I8Vd18jtTXjKkCWsPy2DPBM7zXVbelhrXzNKskxg= +github.com/fermyon/spin/sdk/go v0.5.0/go.mod h1:ARV2oVtnUCykLM+xCBZq8MQrCZddzb3JbeBettYv1S0= diff --git a/templates/redis-rust/content/Cargo.toml b/templates/redis-rust/content/Cargo.toml index 37ef267e0a..c2429503b5 100644 --- a/templates/redis-rust/content/Cargo.toml +++ b/templates/redis-rust/content/Cargo.toml @@ -16,7 +16,7 @@ bytes = "1" # Logging log = { version = "0.4", default-features = false } # The Spin SDK. -spin-sdk = { git = "https://github.com/fermyon/spin", tag = "v0.4.0" } +spin-sdk = { git = "https://github.com/fermyon/spin", tag = "v0.5.0" } # Crate that generates Rust Wasm bindings from a WebAssembly interface. wit-bindgen-rust = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } diff --git a/tests/http/simple-spin-rust/Cargo.lock b/tests/http/simple-spin-rust/Cargo.lock index 3b11a97c31..f00cffee25 100644 --- a/tests/http/simple-spin-rust/Cargo.lock +++ b/tests/http/simple-spin-rust/Cargo.lock @@ -143,7 +143,7 @@ dependencies = [ [[package]] name = "spin-sdk" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "bytes", From 0fb6ead97f056aba8b2b96a03d34991fb3d8cf68 Mon Sep 17 00:00:00 2001 From: Kate Goldenring Date: Fri, 2 Sep 2022 13:21:40 -0700 Subject: [PATCH 007/163] fix: add integration tests and code clean up Signed-off-by: Kate Goldenring Co-authored-by: Karthik Ganeshram --- Cargo.lock | 2 +- crates/plugins/Cargo.toml | 4 - crates/plugins/src/git.rs | 22 +++--- crates/plugins/src/install.rs | 97 ++++++++++++++++-------- crates/plugins/src/lib.rs | 14 +++- crates/plugins/src/plugin_manifest.rs | 36 ++++++++- crates/plugins/src/prompt.rs | 10 ++- crates/plugins/src/uninstall.rs | 7 ++ crates/plugins/src/version_check.rs | 3 +- src/commands.rs | 2 +- src/commands/external.rs | 12 ++- src/commands/plugins.rs | 79 +++++++++++--------- tests/integration.rs | 102 ++++++++++++++++++++++++++ tests/plugin/README.md | 10 +++ tests/plugin/example | 3 + tests/plugin/example.tar.gz | Bin 0 -> 192 bytes 16 files changed, 312 insertions(+), 91 deletions(-) create mode 100644 tests/plugin/README.md create mode 100755 tests/plugin/example create mode 100644 tests/plugin/example.tar.gz diff --git a/Cargo.lock b/Cargo.lock index 378aec17d9..fb1171743a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3470,7 +3470,7 @@ dependencies = [ "flate2", "log", "reqwest", - "semver 1.0.6", + "semver 1.0.13", "serde", "serde_json", "sha2 0.10.3", diff --git a/crates/plugins/Cargo.toml b/crates/plugins/Cargo.toml index 6bd89d6ff5..46e2d0cc56 100644 --- a/crates/plugins/Cargo.toml +++ b/crates/plugins/Cargo.toml @@ -3,8 +3,6 @@ name = "spin-plugins" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] anyhow = "1.0" bytes = "1.1" @@ -19,5 +17,3 @@ tar = "0.4.38" tempfile = "3.3.0" tokio = { version = "1.10", features = [ "fs", "process", "rt", "macros" ] } url = "2.2.2" - - diff --git a/crates/plugins/src/git.rs b/crates/plugins/src/git.rs index 4e42f3e1e5..c1d614565f 100644 --- a/crates/plugins/src/git.rs +++ b/crates/plugins/src/git.rs @@ -3,15 +3,21 @@ use std::path::PathBuf; use tokio::process::Command; use url::Url; -const PLUGINS_REPO_BRANCH: &str = "main"; +const DEFAULT_BRANCH: &str = "main"; +/// Enables cloning and fetching the latest of a git repository to a local +/// directory. pub struct GitSource { + /// Address to remote git repository. source_url: Url, + /// Branch to clone/fetch. branch: String, + /// Destination to clone repository into. local_repo_dir: PathBuf, } impl GitSource { + /// Creates a new git source pub fn new( source_url: &Url, branch: Option, @@ -19,11 +25,12 @@ impl GitSource { ) -> Result { Ok(Self { source_url: source_url.clone(), - branch: branch.unwrap_or_else(|| PLUGINS_REPO_BRANCH.to_owned()), + branch: branch.unwrap_or_else(|| DEFAULT_BRANCH.to_owned()), local_repo_dir, }) } + /// Clones a contents of a git repository to a local directory pub async fn clone(&self) -> Result<()> { let mut git = Command::new("git"); git.args([ @@ -36,10 +43,7 @@ impl GitSource { ]); let clone_result = git.output().await?; match clone_result.status.success() { - true => { - println!("Cloned Repository Successfully!"); - Ok(()) - } + true => Ok(()), false => Err(anyhow!( "Error cloning Git repo {}: {}", self.source_url, @@ -49,15 +53,13 @@ impl GitSource { } } + /// Fetches the latest changes from the source repository pub async fn pull(&self) -> Result<()> { let mut git = Command::new("git"); git.args(["-C", &self.local_repo_dir.to_string_lossy(), "pull"]); let pull_result = git.output().await?; match pull_result.status.success() { - true => { - println!("Updated repository successfully"); - Ok(()) - } + true => Ok(()), false => Err(anyhow!( "Error updating Git repo at {:?}: {}", self.local_repo_dir, diff --git a/crates/plugins/src/install.rs b/crates/plugins/src/install.rs index 06390fbfec..8b7eb78a75 100644 --- a/crates/plugins/src/install.rs +++ b/crates/plugins/src/install.rs @@ -1,13 +1,13 @@ use crate::{ get_manifest_file_name, get_manifest_file_name_version, version_check::{assert_supported_version, get_plugin_manifest}, - PLUGIN_MANIFESTS_DIRECTORY_NAME, + PLUGIN_MANIFESTS_DIRECTORY_NAME, SPIN_INTERNAL_COMMANDS, }; use super::git::GitSource; use super::plugin_manifest::{Os, PluginManifest}; use super::prompt::Prompter; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use flate2::read::GzDecoder; use semver::Version; use std::{ @@ -19,23 +19,32 @@ use tar::Archive; use tempfile::{tempdir, TempDir}; use url::Url; -/// Name of the subdirectory that contains the installed plugin JSON manifests +// Name of directory that contains the cloned centralized Spin plugins +// repository const PLUGINS_REPO_LOCAL_DIRECTORY: &str = ".spin-plugins"; +// Name of directory containing the installed manifests const PLUGINS_REPO_MANIFESTS_DIRECTORY: &str = "manifests"; -/// Anticipated url scheme prefix in manifest to indicate local path to plugin binary +// Url scheme prefix of a plugin that is installed from a local source const URL_FILE_SCHEME: &str = "file"; +/// Location of manifest of the plugin to be installed. pub enum ManifestLocation { + /// Plugin manifest can be copied from a local path. Local(PathBuf), + /// Plugin manifest should be pulled from a specific address. Remote(Url), + /// Plugin manifest lives in the centralized plugins repository PluginsRepository(PluginInfo), } +/// Information about the plugin manifest that should be fetched from the +/// centralized Spin plugins repository. pub struct PluginInfo { name: String, repo_url: Url, version: Option, } + impl PluginInfo { pub fn new(name: &str, repo_url: Url, version: Option) -> Self { Self { @@ -46,6 +55,7 @@ impl PluginInfo { } } +/// Retrieves the appropriate plugin manifest and installs the Spin plugin pub struct PluginInstaller { manifest_location: ManifestLocation, plugins_dir: PathBuf, @@ -68,11 +78,16 @@ impl PluginInstaller { } } + /// Installs a Spin plugin. First attempts to retrieve the plugin manifest. + /// If installing a plugin from the centralized Spin plugins repository, it + /// fetches the latest contents of the repository and searches for the + /// appropriately named and versioned plugin manifest. Parses the plugin + /// manifest to get the appropriate source for the machine OS and + /// architecture. Verifies the checksum of the source, unpacks and installs + /// it into the plugins directory. pub async fn install(&self) -> Result<()> { - // TODO: Potentially handle errors to give useful error messages let plugin_manifest: PluginManifest = match &self.manifest_location { ManifestLocation::Remote(url) => { - // Remote manifest source is provided log::info!("Pulling manifest for plugin from {}", url); reqwest::get(url.as_ref()) .await? @@ -80,9 +95,9 @@ impl PluginInstaller { .await? } ManifestLocation::Local(path) => { - // Local manifest source is provided log::info!("Pulling manifest for plugin from {:?}", path); - let file = File::open(path)?; + let file = File::open(path) + .map_err(|_| anyhow!("The local manifest could not be opened"))?; serde_json::from_reader(file)? } ManifestLocation::PluginsRepository(info) => { @@ -104,6 +119,9 @@ impl PluginInstaller { { git_source.clone().await?; } else { + // TODO: consider moving this to a separate `spin plugin + // update` subcommand rather than always updating the + // repository on each install. git_source.pull().await?; } let file = File::open( @@ -128,15 +146,27 @@ impl PluginInstaller { } }; + // Disallow installing plugins with the same name as spin internal + // subcommands + if SPIN_INTERNAL_COMMANDS + .iter() + .any(|&s| s == plugin_manifest.name()) + { + bail!( + "Trying to install a plugin with the same name '{}' as an internal plugin", + plugin_manifest.name() + ); + } + // Disallow downgrades and reinstalling identical plugins - if let Ok(installed) = get_plugin_manifest(&plugin_manifest.name, &self.plugins_dir) { + if let Ok(installed) = get_plugin_manifest(&plugin_manifest.name(), &self.plugins_dir) { if installed.version > plugin_manifest.version || installed == plugin_manifest { - return Err(anyhow!( + bail!( "plugin {} already installed with version {} but attempting to install same or older version ({})", - installed.name, + installed.name(), installed.version, plugin_manifest.version, - )); + ); } } @@ -149,22 +179,23 @@ impl PluginInstaller { } else if cfg!(target_os = "macos") { Os::Osx } else { - return Err(anyhow!("This plugin is not supported on this OS")); + bail!("This plugin is not supported on this OS"); }; - // TODO: Add logic for architecture as well + let arch = std::env::consts::ARCH; let plugin_package = plugin_manifest .packages .iter() - .find(|p| p.os == os) - .ok_or_else(|| anyhow!("This plugin does not support this OS"))?; + .find(|p| p.os == os && p.arch.to_string() == arch) + .ok_or_else(|| anyhow!("This plugin does not support this OS or architecture"))?; let target = plugin_package.url.to_owned(); - // Ask for user confirmation if not overridden with CLI option + // Ask for user confirmation to install if not overridden with CLI + // option if !self.yes_to_all - && !Prompter::new(&plugin_manifest.name, &plugin_manifest.license, &target)?.run()? + && !Prompter::new(&plugin_manifest.name(), &plugin_manifest.license, &target)?.run()? { // User has requested to not install package, returning early - println!("Plugin {} will not be installed", plugin_manifest.name); + println!("Plugin {} will not be installed", plugin_manifest.name()); return Ok(()); } let target_url = Url::parse(&target)?; @@ -172,15 +203,20 @@ impl PluginInstaller { let plugin_tarball_path = match target_url.scheme() { URL_FILE_SCHEME => PathBuf::from(target_url.path()), _ => { - PluginInstaller::download_plugin(&plugin_manifest.name, &temp_dir, &target).await? + PluginInstaller::download_plugin(&plugin_manifest.name(), &temp_dir, &target) + .await? } }; self.verify_checksum(&plugin_tarball_path, &plugin_package.sha256)?; - self.untar_plugin(&plugin_tarball_path, &plugin_manifest.name)?; + self.untar_plugin(&plugin_tarball_path, &plugin_manifest.name())?; + // Save manifest to installed plugins directory self.add_to_manifest_dir(&plugin_manifest)?; - log::info!("Plugin [{}] installed successfully", plugin_manifest.name); + println!( + "Plugin [{}] was installed successfully!", + plugin_manifest.name() + ); Ok(()) } @@ -191,7 +227,6 @@ impl PluginInstaller { let tar = GzDecoder::new(tar_gz); // Get plugin from tarball let mut archive = Archive::new(tar); - // TODO: this is unix only. Look into whether permissions are preserved archive.set_preserve_permissions(true); // Create subdirectory in plugins directory for this plugin let plugin_sub_dir = self.plugins_dir.join(plugin_name); @@ -202,7 +237,7 @@ impl PluginInstaller { } async fn download_plugin(name: &str, temp_dir: &TempDir, target_url: &str) -> Result { - log::info!( + log::trace!( "Trying to get tar file for plugin {} from {}", name, target_url @@ -217,25 +252,27 @@ impl PluginInstaller { Ok(plugin_file) } - // Validate checksum of downloaded content with checksum from Index fn verify_checksum(&self, plugin_file: &PathBuf, checksum: &str) -> Result<()> { let binary_sha256 = file_digest_string(plugin_file).expect("failed to get sha for parcel"); let verification_sha256 = checksum; if binary_sha256 == verification_sha256 { - println!("Package verified successfully"); + log::info!("Package checksum verified successfully"); Ok(()) } else { - Err(anyhow!("Could not validate Checksum")) + Err(anyhow!( + "Could not validate Checksum, aborting installation" + )) } } - fn add_to_manifest_dir(&self, plugin: &PluginManifest) -> Result<()> { + fn add_to_manifest_dir(&self, plugin_manifest: &PluginManifest) -> Result<()> { let manifests_dir = self.plugins_dir.join(PLUGIN_MANIFESTS_DIRECTORY_NAME); fs::create_dir_all(&manifests_dir)?; serde_json::to_writer( - &File::create(manifests_dir.join(get_manifest_file_name(&plugin.name)))?, - plugin, + &File::create(manifests_dir.join(get_manifest_file_name(&plugin_manifest.name())))?, + plugin_manifest, )?; + log::trace!("Added manifest for {}", &plugin_manifest.name()); Ok(()) } } diff --git a/crates/plugins/src/lib.rs b/crates/plugins/src/lib.rs index 2a396a1cbf..cd8e2dfc76 100644 --- a/crates/plugins/src/lib.rs +++ b/crates/plugins/src/lib.rs @@ -1,11 +1,23 @@ mod git; -// TODO: just export PluginInstaller pub mod install; mod plugin_manifest; mod prompt; pub mod uninstall; pub mod version_check; +/// List of Spin internal subcommands +pub(crate) const SPIN_INTERNAL_COMMANDS: [&str; 9] = [ + "templates", + "up", + "new", + "bindle", + "deploy", + "build", + "plugin", + "trigger", + "external", +]; + /// Directory where the manifests of installed plugins are stored. pub const PLUGIN_MANIFESTS_DIRECTORY_NAME: &str = "manifests"; diff --git a/crates/plugins/src/plugin_manifest.rs b/crates/plugins/src/plugin_manifest.rs index 8a0e4b4e2a..1daaf9b2b3 100644 --- a/crates/plugins/src/plugin_manifest.rs +++ b/crates/plugins/src/plugin_manifest.rs @@ -1,27 +1,49 @@ use serde::{Deserialize, Serialize}; +/// Expected schema of a plugin manifest. Should match the latest Spin plugin +/// manifest JSON schema: +/// https://github.com/fermyon/spin-plugins/tree/main/json-schema #[derive(Serialize, Debug, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub(crate) struct PluginManifest { - pub name: String, + /// Name of the plugin. + name: String, + /// Option description of the plugin. #[serde(default, skip_serializing_if = "Option::is_none")] description: Option, + /// Optional address to the homepage of the plugin producer. #[serde(default, skip_serializing_if = "Option::is_none")] homepage: Option, + /// Version of the plugin. pub version: String, + /// Versions of Spin that the plugin is compatible with. pub spin_compatibility: String, + /// License of the plugin. pub license: String, + /// Points to source package[s] of the plugin.. pub packages: Vec, } +impl PluginManifest { + pub fn name(&self) -> String { + self.name.to_lowercase() + } +} + +/// Describes compatibility and location of a plugin source. #[derive(Serialize, Debug, Deserialize, PartialEq)] pub(crate) struct PluginPackage { + /// Compatible OS. pub os: Os, + /// Compatible architecture. pub arch: Architecture, + /// Address to fetch the plugin source tar file. pub url: String, + /// Checksum to verify the plugin before installation. pub sha256: String, } +/// Describes the compatible OS of a plugin #[derive(Serialize, Debug, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub(crate) enum Os { @@ -30,14 +52,24 @@ pub(crate) enum Os { Windows, } +/// Describes the compatible architecture of a plugin #[derive(Serialize, Debug, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub(crate) enum Architecture { Amd64, Aarch64, + Arm, } -// TODO: create licenses enum +impl ToString for Architecture { + fn to_string(&self) -> String { + match self { + Self::Amd64 => "x86_64".to_string(), + Self::Aarch64 => "aarch64".to_string(), + Self::Arm => "arm".to_string(), + } + } +} #[cfg(test)] mod test { diff --git a/crates/plugins/src/prompt.rs b/crates/plugins/src/prompt.rs index 820442e225..2d7b63f915 100644 --- a/crates/plugins/src/prompt.rs +++ b/crates/plugins/src/prompt.rs @@ -1,8 +1,8 @@ -/// Prompts user as to whether they trust the source of the plugin and -/// want to proceed with installation use anyhow::Result; use std::io; +/// Prompts user as to whether they trust the source of the plugin and +/// want to proceed with installation. pub(crate) struct Prompter { plugin_name: String, plugin_license: String, @@ -10,6 +10,7 @@ pub(crate) struct Prompter { } impl Prompter { + /// Creates a new prompter pub fn new(plugin_name: &str, plugin_license: &str, source_url: &str) -> Result { Ok(Self { plugin_name: plugin_name.to_string(), @@ -17,6 +18,7 @@ impl Prompter { source_url: source_url.to_string(), }) } + fn print_prompt(&self) { println!( "Installing plugin {} with license {} from {}\n", @@ -30,13 +32,13 @@ impl Prompter { io::stdin().read_line(&mut resp)?; Ok(self.parse_response(&mut resp)) } + fn parse_response(&self, resp: &mut str) -> bool { let resp = resp.trim().to_lowercase(); resp.eq("yes") || resp.eq("y") - // TODO: consider checking for invalid response } - // Returns whether or not the user would like to proceed with the installation + /// Returns whether or not the user would like to proceed with the installation of a plugin. pub(crate) fn run(&self) -> Result { self.print_prompt(); self.are_you_sure() diff --git a/crates/plugins/src/uninstall.rs b/crates/plugins/src/uninstall.rs index eef879f499..5771abec75 100644 --- a/crates/plugins/src/uninstall.rs +++ b/crates/plugins/src/uninstall.rs @@ -1,8 +1,12 @@ use crate::{get_manifest_file_name, PLUGIN_MANIFESTS_DIRECTORY_NAME}; use anyhow::Result; use std::{fs, path::PathBuf}; + +/// Settings for uninstalling a plugin. pub struct PluginUninstaller { + /// Name of plugin to be uninstalled. plugin_name: String, + /// Path to the directory where plugins are installed. plugins_dir: PathBuf, } @@ -13,6 +17,9 @@ impl PluginUninstaller { plugins_dir, } } + + /// Uninstalls a plugin with a given name, removing it and it's manifest + /// from the local plugins directory. pub fn run(&self) -> Result<()> { // Check if plugin is installed let manifest_file = self diff --git a/crates/plugins/src/version_check.rs b/crates/plugins/src/version_check.rs index 86fb4eaf43..b165cb820e 100644 --- a/crates/plugins/src/version_check.rs +++ b/crates/plugins/src/version_check.rs @@ -6,7 +6,6 @@ use semver::{Version, VersionReq}; use std::{fs::File, path::Path}; /// Checks whether the plugin supports the currently running version of Spin. -// TODO: check whether on main or canary (aka beyond the specified version). pub fn assert_supported_version(spin_version: &str, supported: &str) -> Result<()> { let supported = VersionReq::parse(supported).map_err(|e| { anyhow!( @@ -37,6 +36,8 @@ pub(crate) fn get_plugin_manifest(plugin_name: &str, plugins_dir: &Path) -> Resu Ok(manifest) } +/// Verifies that a plugin is compatible with the currently running version of Spin +/// by fetching it's manifest and assessing it's `spinCompatibility`. pub fn check_plugin_spin_compatibility( plugin_name: &str, spin_version: &str, diff --git a/src/commands.rs b/src/commands.rs index 9badc7abe0..0da8569c71 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -6,7 +6,7 @@ pub mod bindle; pub mod build; /// Command for deploying a Spin app to Hippo pub mod deploy; -/// commands for external subcommands +/// Commands for external subcommands (i.e. plugins) pub mod external; /// Command for creating a new application. pub mod new; diff --git a/src/commands/external.rs b/src/commands/external.rs index bb59cb2b6f..b439e027db 100644 --- a/src/commands/external.rs +++ b/src/commands/external.rs @@ -1,17 +1,23 @@ +use crate::commands::plugins::get_spin_plugins_directory; + use anyhow::{anyhow, Result}; use spin_plugins::version_check::check_plugin_spin_compatibility; use std::{collections::HashMap, env, path::Path}; use tokio::process::Command; use tracing::log; -use crate::commands::plugins::get_spin_plugins_directory; - +/// Executes a Spin plugin as a subprocess, expecting the first argument to +/// indicate the plugin to execute. Passes all subsequent arguments on to the +/// subprocess. pub async fn execute_external_subcommand(args: Vec) -> Result<()> { let plugin_name = args.first().ok_or_else(|| anyhow!("Expected subcommand"))?; let plugins_dir = get_spin_plugins_directory()?; check_plugin_spin_compatibility(plugin_name, env!("VERGEN_BUILD_SEMVER"), &plugins_dir)?; let path = plugins_dir.join(plugin_name); - let binary = path.join(plugin_name); + let mut binary = path.join(plugin_name); + if cfg!(target_os = "windows") { + binary.set_extension("exe"); + } let mut command = Command::new(binary); if args.len() > 1 { command.args(&args[1..]); diff --git a/src/commands/plugins.rs b/src/commands/plugins.rs index e93b42a5b1..02729155d2 100644 --- a/src/commands/plugins.rs +++ b/src/commands/plugins.rs @@ -12,23 +12,20 @@ use url::Url; const SPIN_PLUGINS_REPO: &str = "https://github.com/fermyon/spin-plugins/"; -/// Install/uninstall plugins +/// Install/uninstall Spin plugins. #[derive(Subcommand, Debug)] pub enum PluginCommands { - /// Install plugin from the Spin plugin repository. + /// Install plugin from a manifest. /// - /// The binary or .wasm file of the plugin is copied to the local Spin plugins directory - /// TODO: consider the ability to install multiple plugins + /// The binary file and manifest of the plugin is copied to the local Spin + /// plugins directory. Install(Install), /// Remove a plugin from your installation. Uninstall(Uninstall), - /// Upgrade one or all plugins + /// Upgrade one or all plugins. Upgrade(Upgrade), - // TODO: consider Search command - - // TODO: consider List command } impl PluginCommands { @@ -52,7 +49,8 @@ pub struct Install { required_unless_present_any = ["REMOTE_PLUGIN_MANIFEST", "LOCAL_PLUGIN_MANIFEST"], )] pub name: Option, - /// source of local manifest file + + /// Path to local plugin manifest. #[clap( name = "LOCAL_PLUGIN_MANIFEST", short = 'f', @@ -61,7 +59,8 @@ pub struct Install { conflicts_with = "PLUGIN_NAME" )] pub local_manifest_src: Option, - /// source of remote manifest file + + /// Path to remote plugin manifest. #[clap( name = "REMOTE_PLUGIN_MANIFEST", short = 'u', @@ -70,10 +69,13 @@ pub struct Install { conflicts_with = "PLUGIN_NAME" )] pub remote_manifest_src: Option, - /// skips prompt to accept the installation of the plugin. + + /// Skips prompt to accept the installation of the plugin. #[clap(short = 'y', long = "yes", takes_value = false)] pub yes_to_all: bool, - /// specify particular version of plugin to install from centralized repository + + /// Specific version of a plugin to be install from the centralized plugins + /// repository. #[clap( long = "version", short = 'v', @@ -81,14 +83,12 @@ pub struct Install { conflicts_with = "LOCAL_PLUGIN_MANIFEST", requires("PLUGIN_NAME") )] - /// Specify a particular version of the plugin to be installed from the Centralized Repository pub version: Option, } impl Install { pub async fn run(self) -> Result<()> { let manifest_location = match (self.local_manifest_src, self.remote_manifest_src, self.name) { - // TODO: move all this parsing into clap to catch input errors. (Some(path), None, None) => ManifestLocation::Local(path), (None, Some(url), None) => ManifestLocation::Remote(url), (None, None, Some(name)) => ManifestLocation::PluginsRepository(PluginInfo::new(&name, Url::parse(SPIN_PLUGINS_REPO)?, self.version)), @@ -106,14 +106,11 @@ impl Install { } } -/// Remove the specified plugin +/// Uninstalls specified plugin. #[derive(Parser, Debug)] pub struct Uninstall { /// Name of Spin plugin. pub name: String, - // TODO: think about how to handle breaking changes - // #[structopt(long = "update")] - // pub update: bool, } impl Uninstall { @@ -132,51 +129,62 @@ pub struct Upgrade { required_unless_present_any = ["ALL"], )] pub name: Option, - /// Upgrade all plugins + + /// Upgrade all plugins. #[clap( short = 'a', long = "all", name = "ALL", conflicts_with = "PLUGIN_NAME", + conflicts_with = "REMOTE_PLUGIN_MANIFEST", + conflicts_with = "LOCAL_PLUGIN_MANIFEST", takes_value = false )] pub all: bool, - /// Source of local manifest file + + /// Path to local plugin manifest. #[clap( name = "LOCAL_PLUGIN_MANIFEST", short = 'f', long = "file", - conflicts_with = "REMOTE_PLUGIN_MANIFEST", - conflicts_with = "PLUGIN_NAME" + conflicts_with = "REMOTE_PLUGIN_MANIFEST" )] pub local_manifest_src: Option, - /// Source of remote manifest file + + /// Path to remote plugin manifest. #[clap( name = "REMOTE_PLUGIN_MANIFEST", short = 'u', long = "url", - conflicts_with = "LOCAL_PLUGIN_MANIFEST", - conflicts_with = "PLUGIN_NAME" + conflicts_with = "LOCAL_PLUGIN_MANIFEST" )] pub remote_manifest_src: Option, - /// skips prompt to accept the installation of the plugin. + + /// Skips prompt to accept the installation of the plugin[s]. #[clap(short = 'y', long = "yes", takes_value = false)] pub yes_to_all: bool, + + /// Specific version of a plugin to be install from the centralized plugins + /// repository. #[clap( long = "version", short = 'v', conflicts_with = "REMOTE_PLUGIN_MANIFEST", conflicts_with = "LOCAL_PLUGIN_MANIFEST", + conflicts_with = "ALL", requires("PLUGIN_NAME") )] - /// Specify a particular version of the plugin to be installed from the centralized plugin repository pub version: Option, - /// Allow downgrading a plugin's version + + /// Allow downgrading a plugin's version. #[clap(short = 'd', long = "downgrade", takes_value = false)] pub downgrade: bool, } impl Upgrade { + /// Upgrades one or all plugins by reinstalling the latest or a specified + /// version of a plugin. If downgrade is specified, first uninstalls the + /// plugin. pub async fn run(self) -> Result<()> { let plugins_dir = get_spin_plugins_directory()?; let spin_version = env!("VERGEN_BUILD_SEMVER"); @@ -211,7 +219,8 @@ impl Upgrade { .install() .await { - // Ignore plugins that were not installed from the central plugins repository + // Ignore plugins that were not installed from the central + // plugins repository if e.to_string().contains("Could not find plugin") { log::info!( "Could not update {} plugin as DNE in central repository", @@ -231,7 +240,6 @@ impl Upgrade { PluginUninstaller::new(&name, plugins_dir.clone()).run()?; } let manifest_location = match (self.local_manifest_src, self.remote_manifest_src) { - // TODO: move all this parsing into clap to catch input errors. (Some(path), None) => ManifestLocation::Local(path), (None, Some(url)) => ManifestLocation::Remote(url), _ => ManifestLocation::PluginsRepository(PluginInfo::new( @@ -254,11 +262,14 @@ impl Upgrade { } } -/// Gets the path to where Spin plugin are (to be) installed +/// Gets the path to where Spin plugin are installed. pub fn get_spin_plugins_directory() -> anyhow::Result { - let data_dir = dirs::data_local_dir() - .or_else(|| dirs::home_dir().map(|p| p.join(".spin"))) - .ok_or_else(|| anyhow!("Unable to get local data directory or home directory"))?; + let data_dir = match std::env::var("TEST_PLUGINS_DIRECTORY") { + Ok(test_dir) => PathBuf::from(test_dir), + Err(_) => dirs::data_local_dir() + .or_else(|| dirs::home_dir().map(|p| p.join(".spin"))) + .ok_or_else(|| anyhow!("Unable to get local data directory or home directory"))?, + }; let plugins_dir = data_dir.join("spin").join("plugins"); Ok(plugins_dir) } diff --git a/tests/integration.rs b/tests/integration.rs index e8e0450f51..804caccdc2 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -8,11 +8,13 @@ mod integration_tests { }; use std::{ ffi::OsStr, + fs::{self, File}, net::{Ipv4Addr, SocketAddrV4, TcpListener}, path::Path, process::{self, Child, Command}, time::Duration, }; + use tempfile::tempdir; use tokio::{net::TcpStream, time::sleep}; const RUST_HTTP_INTEGRATION_TEST: &str = "tests/http/simple-spin-rust"; @@ -870,6 +872,106 @@ mod integration_tests { Ok(()) } + #[test] + fn test_spin_plugin_install_command() -> Result<()> { + // Create a temporary directory for plugin source and manifests + let temp_dir = tempdir()?; + let dir = temp_dir.path(); + let installed_plugins_dir = dir.join("tmp"); + + // Ensure that spin installs the plugins into the temporary directory + std::env::set_var( + "TEST_PLUGINS_DIRECTORY", + installed_plugins_dir.to_str().unwrap(), + ); + + let path_to_test_dir = std::env::current_dir()?; + // TODO: test on Windows + let plugin_manifest_json_template = r#" + { + "name": "example", + "description": "A description of the plugin.", + "homepage": "www.example.com", + "version": "0.2.0", + "spinCompatibility": ">=0.4, <=5.0", + "license": "MIT", + "packages": [ + { + "os": "linux", + "arch": "amd64", + "url": "file:PATH_TO_TESTS/tests/plugin/example.tar.gz", + "sha256": "57a0d87fcd9900b0122affcb570c5bb878246e2f169f4db377fd055af8e0491e" + }, + { + "os": "osx", + "arch": "aarch64", + "url": "file:PATH_TO_TESTS/tests/plugin/example.tar.gz", + "sha256": "57a0d87fcd9900b0122affcb570c5bb878246e2f169f4db377fd055af8e0491e" + } + ] + }"#; + let plugin_manifest_json = plugin_manifest_json_template + .replace("PATH_TO_TESTS", path_to_test_dir.to_str().unwrap()); + + let manifest_file_path = dir.join("example-plugin-manifest.json"); + fs::write(&manifest_file_path, plugin_manifest_json.as_bytes())?; + + let output_file_path = dir.join("example.txt"); + File::create(&output_file_path)?; + + // Install plugin + let install_args = vec![ + SPIN_BINARY, + "plugin", + "install", + "--file", + manifest_file_path.to_str().unwrap(), + "--yes", + ]; + run(install_args, None)?; + + // Execute example plugin which writes "This is an example Spin plugin!" to a specified file + let execute_args = vec![SPIN_BINARY, "example", output_file_path.to_str().unwrap()]; + run(execute_args, None)?; + + // Verify plugin successfully wrote to output file + let contents = fs::read_to_string(output_file_path)?; + assert_eq!(contents.trim(), "This is an example Spin plugin!"); + + // Upgrade plugin to newer version + let plugin_manifest_json_latest = plugin_manifest_json.replace("0.2.0", "0.2.1"); + fs::write( + dir.join("example-plugin-manifest.json"), + plugin_manifest_json_latest.as_bytes(), + )?; + let upgrade_args = vec![ + SPIN_BINARY, + "plugin", + "upgrade", + "example", + "--file", + manifest_file_path + .to_str() + .ok_or_else(|| anyhow::anyhow!("Cannot convert PathBuf to str"))?, + "--yes", + ]; + run(upgrade_args, None)?; + + // Check plugin version + let installed_manifest = installed_plugins_dir + .join("spin") + .join("plugins") + .join("manifests") + .join("example.json"); + let manifest = fs::read_to_string(installed_manifest)?; + assert!(manifest.contains("0.2.1")); + + // Uninstall plugin + let uninstall_args = vec![SPIN_BINARY, "plugin", "uninstall", "example"]; + run(uninstall_args, None)?; + Ok(()) + } + #[tokio::test] async fn test_build_command() -> Result<()> { do_test_build_command("tests/build/simple").await diff --git a/tests/plugin/README.md b/tests/plugin/README.md new file mode 100644 index 0000000000..89d062608e --- /dev/null +++ b/tests/plugin/README.md @@ -0,0 +1,10 @@ +# Example Plugin + +This `example.sh` script acts as an example Spin plugin for testing Spin plugin functionality. +It is referenced in the `spin plugin` [integration tests](../integration.rs) + +To recreate: + +1. Package and zip it by running `tar czvf example.tar.gz example`. +2. Get checksum: `shasum -a 256 example.tar.gz`. +3. Modify plugin manifest to use the correct checksum. diff --git a/tests/plugin/example b/tests/plugin/example new file mode 100755 index 0000000000..19424bb2ce --- /dev/null +++ b/tests/plugin/example @@ -0,0 +1,3 @@ +#!/bin/bash +FILE=${1:-~/tmp/example.txt} +echo "This is an example Spin plugin!" > $FILE \ No newline at end of file diff --git a/tests/plugin/example.tar.gz b/tests/plugin/example.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..13d8fee36468e5c8fcb8affdc8d1498493cad74d GIT binary patch literal 192 zcmV;x06+g9iwFSvB@tr)152$)%q_@CWuPT6FfcGTHB|u9W)RxI$OJ-zfT5AOf}x4A zfr*ikfw{Snf`K8BZNi{nKvP#iL1}SGViC|O*@@}-IVrf+0hOht;V}o|6$LmwM1ZoQ zeo|(heo|s_2A7+skE^XpwV{=6oqkDffqp6^ZRnL$l+<#iCTHX;D1~HX7ApWzVx9t2 uRw1|`Gf$x)r!+k?Pf Date: Wed, 7 Sep 2022 12:43:06 +1200 Subject: [PATCH 008/163] Validate name when instantiating template Signed-off-by: itowlson --- src/commands/new.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/commands/new.rs b/src/commands/new.rs index 61e6bccf10..2e3c1623e8 100644 --- a/src/commands/new.rs +++ b/src/commands/new.rs @@ -18,6 +18,7 @@ pub struct NewCommand { pub template_id: String, /// The name of the new application or component. + #[clap(value_parser = validate_name)] pub name: String, /// The directory in which to create the new application or component. @@ -125,6 +126,7 @@ fn merge_values(from_file: &mut HashMap, from_cli: &[ParameterVa lazy_static::lazy_static! { static ref PATH_UNSAFE_CHARACTERS: regex::Regex = regex::Regex::new("[^-_.a-zA-Z0-9]").expect("Invalid path safety regex"); + static ref PROJECT_NAME: regex::Regex = regex::Regex::new("^[a-zA-Z].*").expect("Invalid project name regex"); } fn path_safe(text: &str) -> PathBuf { @@ -132,6 +134,14 @@ fn path_safe(text: &str) -> PathBuf { PathBuf::from(path.to_string()) } +fn validate_name(name: &str) -> Result { + if PROJECT_NAME.is_match(name) { + Ok(name.to_owned()) + } else { + Err("Name must start with a letter".to_owned()) + } +} + #[cfg(test)] mod tests { use std::io::Write; @@ -203,4 +213,13 @@ mod tests { merge_values(&mut values, &from_cli); assert_eq!(want, values); } + + #[test] + fn project_names_must_start_with_letter() { + assert_eq!("hello", validate_name("hello").unwrap()); + assert_eq!("Proj123!.456", validate_name("Proj123!.456").unwrap()); + validate_name("123").unwrap_err(); + validate_name("1hello").unwrap_err(); + validate_name("_foo").unwrap_err(); + } } From 0041bae9a15dbdf8449e5f4e1f2d9b8559f73037 Mon Sep 17 00:00:00 2001 From: Opemipo Disu <46439130+coderoflagos@users.noreply.github.com> Date: Thu, 8 Sep 2022 09:37:38 +0100 Subject: [PATCH 009/163] minor corrections Signed-off-by: Opemipo Disu <46439130+coderoflagos@users.noreply.github.com> --- docs/content/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/index.md b/docs/content/index.md index eff2ac0982..37b2353fe6 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -41,7 +41,7 @@ fn hello_world(_req: Request) -> Result {​ #### Spin Manifest Once the code is compiled to a WebAssembly component, it can be referenced in a `spin.toml` -file to create an HTTP application like below. +file to create an HTTP application like what you can see below: ```toml spin_version = "1" @@ -58,8 +58,8 @@ route = "/hello" #### Running a Spin Application -Running this application with the `spin` CLI is as simple as using the `spin up` command. -Because a trigger type of `http` is specified in the `spin.toml`, `spin up` will start +Running this application with the spin CLI is as simple as using the `spin up` command. +Because a trigger type of `http` is specified in the `spin.toml` file, `spin up` will start a web server: ```console From 6209a0242d7e59b6b8b96c6dfcc3cfb0bd3a8210 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 8 Sep 2022 08:17:28 -0400 Subject: [PATCH 010/163] Rename wasi-outbound-http crate to outbound-http Signed-off-by: Lann Martin --- Cargo.lock | 42 ++++++++++++++++----------------- Cargo.toml | 2 +- crates/loader/Cargo.toml | 2 +- crates/loader/src/validation.rs | 2 +- crates/outbound-http/Cargo.toml | 2 +- crates/trigger/Cargo.toml | 2 +- crates/trigger/src/lib.rs | 2 +- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e7f0028dc..6ab76bd8e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2185,6 +2185,24 @@ version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +[[package]] +name = "outbound-http" +version = "0.2.0" +dependencies = [ + "anyhow", + "bytes", + "futures", + "http", + "reqwest", + "spin-engine", + "spin-manifest", + "tokio", + "tracing", + "tracing-futures", + "url", + "wit-bindgen-wasmtime", +] + [[package]] name = "outbound-pg" version = "0.2.0" @@ -3296,6 +3314,7 @@ dependencies = [ "lazy_static", "nix 0.24.2", "openssl", + "outbound-http", "outbound-redis", "path-absolutize", "regex", @@ -3323,7 +3342,6 @@ dependencies = [ "url", "uuid", "vergen", - "wasi-outbound-http", "wasmtime", "which", ] @@ -3415,6 +3433,7 @@ dependencies = [ "glob", "itertools", "lazy_static", + "outbound-http", "path-absolutize", "regex", "reqwest", @@ -3430,7 +3449,6 @@ dependencies = [ "tracing-futures", "tracing-subscriber", "walkdir", - "wasi-outbound-http", ] [[package]] @@ -3602,6 +3620,7 @@ dependencies = [ "dotenvy", "futures", "http", + "outbound-http", "outbound-pg", "outbound-redis", "serde", @@ -3610,7 +3629,6 @@ dependencies = [ "spin-loader", "spin-manifest", "tracing", - "wasi-outbound-http", "wasmtime", ] @@ -4296,24 +4314,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "wasi-outbound-http" -version = "0.2.0" -dependencies = [ - "anyhow", - "bytes", - "futures", - "http", - "reqwest", - "spin-engine", - "spin-manifest", - "tokio", - "tracing", - "tracing-futures", - "url", - "wit-bindgen-wasmtime", -] - [[package]] name = "wasm-bindgen" version = "0.2.82" diff --git a/Cargo.toml b/Cargo.toml index ae1d9c0f3d..bd8b1cc5d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ hippo-openapi = "0.10" hippo = { git = "https://github.com/deislabs/hippo-cli", tag = "v0.16.1" } lazy_static = "1.4.0" nix = { version = "0.24", features = ["signal"] } +outbound-http = { path = "crates/outbound-http" } outbound-redis = { path = "crates/outbound-redis" } path-absolutize = "3.0.11" regex = "1.5.5" @@ -47,7 +48,6 @@ tracing-futures = "0.2" tracing-subscriber = { version = "0.3.7", features = [ "env-filter" ] } url = "2.2.2" uuid = "^1.0" -wasi-outbound-http = { path = "crates/outbound-http" } wasmtime = "0.39.1" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/crates/loader/Cargo.toml b/crates/loader/Cargo.toml index fc496fd957..38ceb39ea5 100644 --- a/crates/loader/Cargo.toml +++ b/crates/loader/Cargo.toml @@ -15,6 +15,7 @@ futures = "0.3.17" glob = "0.3.0" itertools = "0.10.3" lazy_static = "1.4.0" +outbound-http = { path = "../outbound-http" } path-absolutize = "3.0.11" regex = "1.5.4" reqwest = "0.11.9" @@ -30,4 +31,3 @@ tracing = { version = "0.1", features = [ "log" ] } tracing-futures = "0.2" tracing-subscriber = { version = "0.3.7", features = [ "env-filter" ] } walkdir = "2.3.2" -wasi-outbound-http = { path = "../outbound-http" } diff --git a/crates/loader/src/validation.rs b/crates/loader/src/validation.rs index 5ddf4c1fb3..a0f89099cb 100644 --- a/crates/loader/src/validation.rs +++ b/crates/loader/src/validation.rs @@ -16,7 +16,7 @@ pub fn parse_allowed_http_hosts(raw: &Option>) -> Result { if list .iter() - .any(|domain| domain == wasi_outbound_http::ALLOW_ALL_HOSTS) + .any(|domain| domain == outbound_http::ALLOW_ALL_HOSTS) { Ok(AllowedHttpHosts::AllowAll) } else { diff --git a/crates/outbound-http/Cargo.toml b/crates/outbound-http/Cargo.toml index 7c0ed02bcb..917d610af7 100644 --- a/crates/outbound-http/Cargo.toml +++ b/crates/outbound-http/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "wasi-outbound-http" +name = "outbound-http" version = "0.2.0" edition = "2021" authors = ["Fermyon Engineering "] diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml index 2bf1cd96ee..5db3a1c068 100644 --- a/crates/trigger/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -12,6 +12,7 @@ ctrlc = { version = "3.2", features = ["termination"] } dotenvy = "0.15.1" futures = "0.3" http = "0.2" +outbound-http = { path = "../outbound-http" } outbound-redis = { path = "../outbound-redis" } outbound-pg = { path = "../outbound-pg" } serde = "1.0" @@ -20,5 +21,4 @@ spin-engine = { path = "../engine" } spin-loader = { path = "../loader" } spin-manifest = { path = "../manifest" } tracing = { version = "0.1", features = [ "log" ] } -wasi-outbound-http = { path = "../outbound-http" } wasmtime = "0.39.1" \ No newline at end of file diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 76a7608f9f..bb88ab46d2 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -124,7 +124,7 @@ impl TriggerExecutorBuilder { /// Add the default set of host components to the given builder. pub fn add_default_host_components(builder: &mut Builder) -> Result<()> { - builder.add_host_component(wasi_outbound_http::OutboundHttpComponent)?; + builder.add_host_component(outbound_http::OutboundHttpComponent)?; builder.add_host_component(outbound_redis::OutboundRedis { connections: Arc::new(RwLock::new(HashMap::new())), })?; From 260a6dd8307fdf88f0295e6f11f5fbe42693a2c1 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 8 Sep 2022 09:09:59 -0400 Subject: [PATCH 011/163] Enable wasmtime async support This is needed to allow for some future features such as cooperative multi-tasking via fuel or epoch interruption. Signed-off-by: Lann Martin --- Cargo.lock | 4 +- crates/config/Cargo.toml | 7 ++- crates/config/src/host_component.rs | 9 ++- crates/engine/Cargo.toml | 8 ++- crates/engine/src/host_component.rs | 4 +- crates/engine/src/lib.rs | 11 ++-- crates/http/Cargo.toml | 8 ++- crates/http/src/lib.rs | 2 +- crates/http/src/spin.rs | 73 +++++++++++----------- crates/http/src/wagi.rs | 21 ++++--- crates/outbound-http/Cargo.toml | 7 ++- crates/outbound-http/src/host_component.rs | 2 +- crates/outbound-http/src/lib.rs | 46 +++++--------- crates/outbound-pg/Cargo.toml | 8 ++- crates/outbound-pg/src/lib.rs | 12 ++-- crates/outbound-redis/Cargo.toml | 6 +- crates/outbound-redis/src/lib.rs | 16 ++--- crates/redis/Cargo.toml | 6 +- crates/redis/src/lib.rs | 2 +- crates/redis/src/spin.rs | 14 ++--- crates/testing/src/lib.rs | 5 +- crates/trigger/src/lib.rs | 6 +- docs/content/extending-and-embedding.md | 2 +- examples/spin-timer/Cargo.toml | 6 +- examples/spin-timer/src/main.rs | 24 +++---- 25 files changed, 161 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ab76bd8e8..7f1463987c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2196,7 +2196,6 @@ dependencies = [ "reqwest", "spin-engine", "spin-manifest", - "tokio", "tracing", "tracing-futures", "url", @@ -2211,6 +2210,7 @@ dependencies = [ "postgres", "spin-engine", "spin-manifest", + "tokio", "tracing", "wit-bindgen-wasmtime", ] @@ -3353,6 +3353,7 @@ dependencies = [ "anyhow", "serde", "thiserror", + "tokio", "toml", "wit-bindgen-wasmtime", ] @@ -4878,6 +4879,7 @@ version = "0.2.0" source = "git+https://github.com/bytecodealliance/wit-bindgen?rev=cb871cfa1ee460b51eb1d144b175b9aab9c50aba#cb871cfa1ee460b51eb1d144b175b9aab9c50aba" dependencies = [ "anyhow", + "async-trait", "bitflags", "thiserror", "wasmtime", diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index d7edb3c71f..a5f8405d64 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -8,7 +8,12 @@ authors = [ "Fermyon Engineering " ] anyhow = "1.0" serde = { version = "1.0", features = [ "derive" ] } thiserror = "1" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } +tokio = { version = "1", features = [ "rt-multi-thread" ] } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] [dev-dependencies] toml = "0.5" diff --git a/crates/config/src/host_component.rs b/crates/config/src/host_component.rs index 20e7528bc6..9d7f234211 100644 --- a/crates/config/src/host_component.rs +++ b/crates/config/src/host_component.rs @@ -3,9 +3,10 @@ use std::sync::Arc; use crate::{Error, Key, Resolver, TreePath}; mod wit { - wit_bindgen_wasmtime::export!("../../wit/ephemeral/spin-config.wit"); + wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/spin-config.wit"], async: *}); } pub use wit::spin_config::add_to_linker; +use wit_bindgen_wasmtime::async_trait; /// A component configuration interface implementation. pub struct ComponentConfig { @@ -26,11 +27,13 @@ impl ComponentConfig { } } +#[async_trait] impl wit::spin_config::SpinConfig for ComponentConfig { - fn get_config(&mut self, key: &str) -> Result { + async fn get_config(&mut self, key: &str) -> Result { let key = Key::new(key)?; let path = &self.component_root + key; - Ok(self.resolver.resolve(&path)?) + // TODO(lann): Make resolve async + tokio::task::block_in_place(|| Ok(self.resolver.resolve(&path)?)) } } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 03a1f0a108..2353f1c77c 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -17,9 +17,11 @@ tracing = { version = "0.1", features = [ "log" ] } tracing-futures = "0.2" wasi-cap-std-sync = "0.39.1" wasi-common = "0.39.1" -wasmtime = "0.39.1" +wasmtime = { version = "0.39.1", features = [ "async" ] } wasmtime-wasi = "0.39.1" cap-std = "0.24.1" -[dev-dependencies] -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } +[dev-dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = [ "async" ] diff --git a/crates/engine/src/host_component.rs b/crates/engine/src/host_component.rs index 57b4b09d42..059fdeaad2 100644 --- a/crates/engine/src/host_component.rs +++ b/crates/engine/src/host_component.rs @@ -12,7 +12,7 @@ pub trait HostComponent: Send + Sync { type State: Any + Send; /// Add this component to the given Linker, using the given runtime state-getting handle. - fn add_to_linker( + fn add_to_linker( linker: &mut Linker>, state_handle: HostComponentsStateHandle, ) -> Result<()>; @@ -30,7 +30,7 @@ pub(crate) struct HostComponents { } impl HostComponents { - pub(crate) fn insert( + pub(crate) fn insert( &mut self, linker: &mut Linker>, host_component: Component, diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 43bf7498ea..4f9b31b320 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -59,7 +59,8 @@ pub struct Engine(wasmtime::Engine); impl Engine { /// Create a new engine and initialize it with the given config. - pub fn new(config: wasmtime::Config) -> Result { + pub fn new(mut config: wasmtime::Config) -> Result { + config.async_support(true); Ok(Self(wasmtime::Engine::new(&config)?)) } @@ -80,7 +81,7 @@ pub struct Builder { host_components: HostComponents, } -impl Builder { +impl Builder { /// Creates a new instance of the execution builder. pub fn new(config: ExecutionContextConfiguration) -> Result> { Self::with_engine(config, Engine::new(Default::default())?) @@ -216,10 +217,10 @@ pub struct ExecutionContext { host_components: Arc, } -impl ExecutionContext { +impl ExecutionContext { /// Creates a store for a given component given its configuration and runtime data. #[instrument(skip(self, data, io))] - pub fn prepare_component( + pub async fn prepare_component( &self, component: &str, data: Option, @@ -234,7 +235,7 @@ impl ExecutionContext { }; let mut store = self.store(component, data, io, env, args)?; - let instance = component.pre.instantiate(&mut store)?; + let instance = component.pre.instantiate_async(&mut store).await?; Ok((store, instance)) } diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index 7caf3cafac..beb3f5a5ff 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -39,9 +39,13 @@ tracing-subscriber = { version = "0.3.7", features = ["env-filter"] } url = "2.2" wasi-cap-std-sync = "0.39.1" wasi-common = "0.39.1" -wasmtime = "0.39.1" +wasmtime = { version = "0.39.1", features = ["async"] } wasmtime-wasi = "0.39.1" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] [dev-dependencies] criterion = { version = "0.3.5", features = ["async_tokio"] } diff --git a/crates/http/src/lib.rs b/crates/http/src/lib.rs index d4a32d4d7a..77a187ba03 100644 --- a/crates/http/src/lib.rs +++ b/crates/http/src/lib.rs @@ -33,7 +33,7 @@ use crate::{ wagi::WagiHttpExecutor, }; -wit_bindgen_wasmtime::import!("../../wit/ephemeral/spin-http.wit"); +wit_bindgen_wasmtime::import!({paths: ["../../wit/ephemeral/spin-http.wit"], async: *}); type ExecutionContext = spin_engine::ExecutionContext; type RuntimeContext = spin_engine::RuntimeContext; diff --git a/crates/http/src/spin.rs b/crates/http/src/spin.rs index e2c2913740..123f13f96c 100644 --- a/crates/http/src/spin.rs +++ b/crates/http/src/spin.rs @@ -7,7 +7,6 @@ use async_trait::async_trait; use hyper::{Body, Request, Response}; use spin_engine::io::ModuleIoRedirects; use std::{net::SocketAddr, str, str::FromStr}; -use tokio::task::spawn_blocking; use tracing::log; use wasmtime::{Instance, Store}; @@ -33,8 +32,9 @@ impl HttpExecutor for SpinHttpExecutor { let mior = ModuleIoRedirects::new(follow); - let (store, instance) = - engine.prepare_component(component, None, Some(mior.pipes), None, None)?; + let (store, instance) = engine + .prepare_component(component, None, Some(mior.pipes), None, None) + .await?; let resp_result = Self::execute_impl(store, instance, base, raw_route, req) .await @@ -76,50 +76,47 @@ impl SpinHttpExecutor { let (parts, bytes) = req.into_parts(); let bytes = hyper::body::to_bytes(bytes).await?.to_vec(); - let res = spawn_blocking(move || -> Result { - let method = Self::method(&parts.method); - - let headers: Vec<(&str, &str)> = headers - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect(); - - // Preparing to remove the params field. We are leaving it in place for now - // to avoid breaking the ABI, but no longer pass or accept values in it. - // https://github.com/fermyon/spin/issues/663 - let params = vec![]; - - let body = Some(&bytes[..]); - let uri = match parts.uri.path_and_query() { - Some(u) => u.to_string(), - None => parts.uri.to_string(), - }; - - let req = crate::spin_http::Request { - method, - uri: &uri, - headers: &headers, - params: ¶ms, - body, - }; - - Ok(engine.handle_http_request(&mut store, req)?) - }) - .await??; - - if res.status < 100 || res.status > 600 { + let method = Self::method(&parts.method); + + let headers: Vec<(&str, &str)> = headers + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + + // Preparing to remove the params field. We are leaving it in place for now + // to avoid breaking the ABI, but no longer pass or accept values in it. + // https://github.com/fermyon/spin/issues/663 + let params = vec![]; + + let body = Some(&bytes[..]); + let uri = match parts.uri.path_and_query() { + Some(u) => u.to_string(), + None => parts.uri.to_string(), + }; + + let req = crate::spin_http::Request { + method, + uri: &uri, + headers: &headers, + params: ¶ms, + body, + }; + + let resp = engine.handle_http_request(&mut store, req).await?; + + if resp.status < 100 || resp.status > 600 { log::error!("malformed HTTP status code"); return Ok(Response::builder() .status(http::StatusCode::INTERNAL_SERVER_ERROR) .body(Body::empty())?); }; - let mut response = http::Response::builder().status(res.status); + let mut response = http::Response::builder().status(resp.status); if let Some(headers) = response.headers_mut() { - Self::append_headers(headers, res.headers)?; + Self::append_headers(headers, resp.headers)?; } - let body = match res.body { + let body = match resp.body { Some(b) => Body::from(b), None => Body::empty(), }; diff --git a/crates/http/src/wagi.rs b/crates/http/src/wagi.rs index 4d2f27b6ce..d919edb46a 100644 --- a/crates/http/src/wagi.rs +++ b/crates/http/src/wagi.rs @@ -12,7 +12,6 @@ use std::{ net::SocketAddr, sync::{Arc, RwLock, RwLockReadGuard}, }; -use tokio::task::spawn_blocking; use tracing::log; use wasi_common::pipe::{ReadPipe, WritePipe}; @@ -84,13 +83,15 @@ impl HttpExecutor for WagiHttpExecutor { headers.insert(keys[1].to_string(), val); } - let (mut store, instance) = engine.prepare_component( - component, - None, - Some(redirects), - Some(headers), - Some(argv.split(' ').map(|s| s.to_owned()).collect()), - )?; + let (mut store, instance) = engine + .prepare_component( + component, + None, + Some(redirects), + Some(headers), + Some(argv.split(' ').map(|s| s.to_owned()).collect()), + ) + .await?; let start = instance .get_func(&mut store, &self.wagi_config.entrypoint) @@ -102,7 +103,7 @@ impl HttpExecutor for WagiHttpExecutor { ) })?; tracing::trace!("Calling Wasm entry point"); - let guest_result = spawn_blocking(move || start.call(&mut store, &[], &mut [])).await; + let guest_result = start.call_async(&mut store, &[], &mut []).await; tracing::info!("Module execution complete"); let log_result = engine.save_output_to_logs(outputs.read(), component, false, true); @@ -111,7 +112,7 @@ impl HttpExecutor for WagiHttpExecutor { // even if the guest code fails. (And when checking, check the guest // result first, so that guest failures are returned in preference to // log failures.) - guest_result?.or_else(ignore_successful_proc_exit_trap)?; + guest_result.or_else(ignore_successful_proc_exit_trap)?; log_result?; let stdout = outputs.stdout.read().unwrap(); diff --git a/crates/outbound-http/Cargo.toml b/crates/outbound-http/Cargo.toml index 917d610af7..731d326809 100644 --- a/crates/outbound-http/Cargo.toml +++ b/crates/outbound-http/Cargo.toml @@ -15,8 +15,11 @@ http = "0.2" reqwest = { version = "0.11", default-features = true, features = [ "json", "blocking" ] } spin-engine = { path = "../engine" } spin-manifest = { path = "../manifest" } -tokio = { version = "1.4.0", features = [ "full" ] } tracing = { version = "0.1", features = [ "log" ] } tracing-futures = "0.2" url = "2.2.1" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] \ No newline at end of file diff --git a/crates/outbound-http/src/host_component.rs b/crates/outbound-http/src/host_component.rs index c3aa252a91..935f501980 100644 --- a/crates/outbound-http/src/host_component.rs +++ b/crates/outbound-http/src/host_component.rs @@ -14,7 +14,7 @@ pub struct OutboundHttpComponent; impl HostComponent for OutboundHttpComponent { type State = OutboundHttp; - fn add_to_linker( + fn add_to_linker( linker: &mut Linker>, data_handle: HostComponentsStateHandle, ) -> Result<()> { diff --git a/crates/outbound-http/src/lib.rs b/crates/outbound-http/src/lib.rs index b7d891fd6d..7894085e53 100644 --- a/crates/outbound-http/src/lib.rs +++ b/crates/outbound-http/src/lib.rs @@ -5,13 +5,13 @@ use http::HeaderMap; use reqwest::{Client, Url}; use spin_manifest::AllowedHttpHosts; use std::str::FromStr; -use tokio::runtime::Handle; use wasi_outbound_http::*; +use wit_bindgen_wasmtime::async_trait; pub use host_component::OutboundHttpComponent; pub use wasi_outbound_http::add_to_linker; -wit_bindgen_wasmtime::export!("../../wit/ephemeral/wasi-outbound-http.wit"); +wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/wasi-outbound-http.wit"], async: *}); pub const ALLOW_ALL_HOSTS: &str = "insecure:allow-all"; @@ -37,8 +37,9 @@ impl OutboundHttp { } } +#[async_trait] impl wasi_outbound_http::WasiOutboundHttp for OutboundHttp { - fn request(&mut self, req: Request) -> Result { + async fn request(&mut self, req: Request<'_>) -> Result { if !self.is_allowed(req.uri)? { tracing::log::info!("Destination not allowed: {}", req.uri); return Err(HttpError::DestinationNotAllowed); @@ -53,35 +54,16 @@ impl wasi_outbound_http::WasiOutboundHttp for OutboundHttp { tracing::log::warn!("HTTP params field is deprecated"); } - match Handle::try_current() { - // If running in a Tokio runtime, spawn a new blocking executor - // that will send the HTTP request, and block on its execution. - // This attempts to avoid any deadlocks from other operations - // already executing on the same executor (compared with just - // blocking on the current one). - Ok(r) => block_on(r.spawn_blocking(move || -> Result { - let client = Client::builder().build().unwrap(); - let res = block_on( - client - .request(method, url) - .headers(headers) - .body(body) - .send(), - ); - let res = log_request_error(res)?; - Response::try_from(res) - })) - .map_err(|_| HttpError::RuntimeError)?, - Err(_) => { - let res = reqwest::blocking::Client::new() - .request(method, url) - .headers(headers) - .body(body) - .send(); - let res = log_request_error(res)?; - Ok(Response::try_from(res)?) - } - } + let client = Client::builder().build().unwrap(); + let res = block_on( + client + .request(method, url) + .headers(headers) + .body(body) + .send(), + ); + let resp = log_request_error(res)?; + Response::try_from(resp).map_err(|_| HttpError::RuntimeError) } } diff --git a/crates/outbound-pg/Cargo.toml b/crates/outbound-pg/Cargo.toml index 769ec0318b..f791a5fc1f 100644 --- a/crates/outbound-pg/Cargo.toml +++ b/crates/outbound-pg/Cargo.toml @@ -11,5 +11,11 @@ anyhow = "1.0" postgres = { version = "0.19.3" } spin-engine = { path = "../engine" } spin-manifest = { path = "../manifest" } +tokio = { version = "1", features = [ "rt" ] } tracing = { version = "0.1", features = [ "log" ] } -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] + diff --git a/crates/outbound-pg/src/lib.rs b/crates/outbound-pg/src/lib.rs index 213706c836..36f232dfcf 100644 --- a/crates/outbound-pg/src/lib.rs +++ b/crates/outbound-pg/src/lib.rs @@ -7,9 +7,9 @@ use spin_engine::{ host_component::{HostComponent, HostComponentsStateHandle}, RuntimeContext, }; -use wit_bindgen_wasmtime::wasmtime::Linker; +use wit_bindgen_wasmtime::{async_trait, wasmtime::Linker}; -wit_bindgen_wasmtime::export!("../../wit/ephemeral/outbound-pg.wit"); +wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/outbound-pg.wit"], async: *}); /// A simple implementation to support outbound pg connection #[derive(Default, Clone)] @@ -18,7 +18,7 @@ pub struct OutboundPg; impl HostComponent for OutboundPg { type State = Self; - fn add_to_linker( + fn add_to_linker( linker: &mut Linker>, state_handle: HostComponentsStateHandle, ) -> anyhow::Result<()> { @@ -33,8 +33,10 @@ impl HostComponent for OutboundPg { } } +// TODO: use spawn_blocking or an async Postgres client +#[async_trait] impl outbound_pg::OutboundPg for OutboundPg { - fn execute( + async fn execute( &mut self, address: &str, statement: &str, @@ -56,7 +58,7 @@ impl outbound_pg::OutboundPg for OutboundPg { Ok(nrow) } - fn query( + async fn query( &mut self, address: &str, statement: &str, diff --git a/crates/outbound-redis/Cargo.toml b/crates/outbound-redis/Cargo.toml index 7e65bea559..491cfb2337 100644 --- a/crates/outbound-redis/Cargo.toml +++ b/crates/outbound-redis/Cargo.toml @@ -14,4 +14,8 @@ spin-engine = { path = "../engine" } spin-manifest = { path = "../manifest" } tracing = { version = "0.1", features = [ "log" ] } tracing-futures = "0.2" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } \ No newline at end of file + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] \ No newline at end of file diff --git a/crates/outbound-redis/src/lib.rs b/crates/outbound-redis/src/lib.rs index 60c114ae49..71f3cd651a 100644 --- a/crates/outbound-redis/src/lib.rs +++ b/crates/outbound-redis/src/lib.rs @@ -11,9 +11,9 @@ use spin_engine::{ host_component::{HostComponent, HostComponentsStateHandle}, RuntimeContext, }; -use wit_bindgen_wasmtime::wasmtime::Linker; +use wit_bindgen_wasmtime::{async_trait, wasmtime::Linker}; -wit_bindgen_wasmtime::export!("../../wit/ephemeral/outbound-redis.wit"); +wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/outbound-redis.wit"], async: *}); /// A simple implementation to support outbound Redis commands. pub struct OutboundRedis { @@ -23,7 +23,7 @@ pub struct OutboundRedis { impl HostComponent for OutboundRedis { type State = Self; - fn add_to_linker( + fn add_to_linker( linker: &mut Linker>, state_handle: HostComponentsStateHandle, ) -> anyhow::Result<()> { @@ -43,8 +43,10 @@ impl HostComponent for OutboundRedis { } } +// TODO: use spawn_blocking or async client methods (redis::aio) +#[async_trait] impl outbound_redis::OutboundRedis for OutboundRedis { - fn publish(&mut self, address: &str, channel: &str, payload: &[u8]) -> Result<(), Error> { + async fn publish(&mut self, address: &str, channel: &str, payload: &[u8]) -> Result<(), Error> { let conn_map = self.get_reused_conn_map(address)?; let mut conn = conn_map .get(address) @@ -55,7 +57,7 @@ impl outbound_redis::OutboundRedis for OutboundRedis { Ok(()) } - fn get(&mut self, address: &str, key: &str) -> Result, Error> { + async fn get(&mut self, address: &str, key: &str) -> Result, Error> { let conn_map = self.get_reused_conn_map(address)?; let mut conn = conn_map .get(address) @@ -66,7 +68,7 @@ impl outbound_redis::OutboundRedis for OutboundRedis { Ok(value) } - fn set(&mut self, address: &str, key: &str, value: &[u8]) -> Result<(), Error> { + async fn set(&mut self, address: &str, key: &str, value: &[u8]) -> Result<(), Error> { let conn_map = self.get_reused_conn_map(address)?; let mut conn = conn_map .get(address) @@ -77,7 +79,7 @@ impl outbound_redis::OutboundRedis for OutboundRedis { Ok(()) } - fn incr(&mut self, address: &str, key: &str) -> Result { + async fn incr(&mut self, address: &str, key: &str) -> Result { let conn_map = self.get_reused_conn_map(address)?; let mut conn = conn_map .get(address) diff --git a/crates/redis/Cargo.toml b/crates/redis/Cargo.toml index ccfbe0fe5f..c97d8563cb 100644 --- a/crates/redis/Cargo.toml +++ b/crates/redis/Cargo.toml @@ -25,7 +25,11 @@ tracing-subscriber = { version = "0.3.7", features = [ "env-filter" ] } wasi-common = "0.39.1" wasmtime = "0.39.1" wasmtime-wasi = "0.39.1" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] [dev-dependencies] spin-testing = { path = "../testing" } diff --git a/crates/redis/src/lib.rs b/crates/redis/src/lib.rs index c7183645b4..a881eec458 100644 --- a/crates/redis/src/lib.rs +++ b/crates/redis/src/lib.rs @@ -12,7 +12,7 @@ use spin_redis::SpinRedisData; use spin_trigger::{cli::NoArgs, TriggerExecutor}; use std::{collections::HashMap, sync::Arc}; -wit_bindgen_wasmtime::import!("../../wit/ephemeral/spin-redis.wit"); +wit_bindgen_wasmtime::import!({paths: ["../../wit/ephemeral/spin-redis.wit"], async: *}); type ExecutionContext = spin_engine::ExecutionContext; type RuntimeContext = spin_engine::RuntimeContext; diff --git a/crates/redis/src/spin.rs b/crates/redis/src/spin.rs index 694f11812e..6fe77f66e1 100644 --- a/crates/redis/src/spin.rs +++ b/crates/redis/src/spin.rs @@ -2,7 +2,6 @@ use crate::{spin_redis::SpinRedis, ExecutionContext, RedisExecutor, RuntimeConte use anyhow::Result; use async_trait::async_trait; use spin_engine::io::ModuleIoRedirects; -use tokio::task::spawn_blocking; use wasmtime::{Instance, Store}; #[derive(Clone)] @@ -25,8 +24,9 @@ impl RedisExecutor for SpinRedisExecutor { let mior = ModuleIoRedirects::new(follow); - let (store, instance) = - engine.prepare_component(component, None, Some(mior.pipes), None, None)?; + let (store, instance) = engine + .prepare_component(component, None, Some(mior.pipes), None, None) + .await?; let result = match Self::execute_impl(store, instance, channel, payload.to_vec()).await { Ok(()) => { @@ -55,13 +55,7 @@ impl SpinRedisExecutor { ) -> Result<()> { let engine = SpinRedis::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?; - let _res = spawn_blocking(move || -> Result { - match engine.handle_redis_message(&mut store, &payload) { - Ok(_) => Ok(crate::spin_redis::Error::Success), - Err(_) => Ok(crate::spin_redis::Error::Error), - } - }) - .await??; + let _res = engine.handle_redis_message(&mut store, &payload).await; Ok(()) } diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 7c4348bc8e..51744b34b4 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -98,7 +98,10 @@ impl TestConfig { } } - pub async fn prepare_builder(&self, app: Application) -> Builder { + pub async fn prepare_builder( + &self, + app: Application, + ) -> Builder { let config = ExecutionContextConfiguration { components: app.components, label: app.info.name, diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index bb88ab46d2..d5901e7449 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -19,7 +19,7 @@ pub trait TriggerExecutor: Sized { type GlobalConfig; type TriggerConfig; type RunConfig; - type RuntimeContext: Default + 'static; + type RuntimeContext: Default + Send + 'static; /// Create a new trigger executor. fn new( @@ -123,7 +123,9 @@ impl TriggerExecutorBuilder { } /// Add the default set of host components to the given builder. -pub fn add_default_host_components(builder: &mut Builder) -> Result<()> { +pub fn add_default_host_components( + builder: &mut Builder, +) -> Result<()> { builder.add_host_component(outbound_http::OutboundHttpComponent)?; builder.add_host_component(outbound_redis::OutboundRedis { connections: Arc::new(RwLock::new(HashMap::new())), diff --git a/docs/content/extending-and-embedding.md b/docs/content/extending-and-embedding.md index d33d2be1c1..c793d6e217 100644 --- a/docs/content/extending-and-embedding.md +++ b/docs/content/extending-and-embedding.md @@ -53,7 +53,7 @@ Let's have a look at building the timer trigger: ```rust // examples/spin-timer/src/main.rs -wit_bindgen_wasmtime::import!("spin-timer.wit"); +wit_bindgen_wasmtime::import!({paths: ["spin-timer.wit"], async: *}); type ExecutionContext = spin_engine::ExecutionContext; /// A custom timer trigger that executes a component on every interval. diff --git a/examples/spin-timer/Cargo.toml b/examples/spin-timer/Cargo.toml index 9aff03f487..597a47cc83 100644 --- a/examples/spin-timer/Cargo.toml +++ b/examples/spin-timer/Cargo.toml @@ -21,4 +21,8 @@ tracing-subscriber = { version = "0.3.7", features = [ "env-filter" ] } wasi-common = "0.39.1" wasmtime = "0.39.1" wasmtime-wasi = "0.39.1" -wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" } + +[dependencies.wit-bindgen-wasmtime] +git = "https://github.com/bytecodealliance/wit-bindgen" +rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" +features = ["async"] diff --git a/examples/spin-timer/src/main.rs b/examples/spin-timer/src/main.rs index 213e2ba9cc..a2c222c704 100644 --- a/examples/spin-timer/src/main.rs +++ b/examples/spin-timer/src/main.rs @@ -6,9 +6,8 @@ use std::{sync::Arc, time::Duration}; use anyhow::Result; use spin_engine::{Builder, ExecutionContextConfiguration}; use spin_manifest::{CoreComponent, ModuleSource, WasmConfig}; -use tokio::task::spawn_blocking; -wit_bindgen_wasmtime::import!("spin-timer.wit"); +wit_bindgen_wasmtime::import!({paths: ["spin-timer.wit"], async: *}); type ExecutionContext = spin_engine::ExecutionContext; @@ -58,21 +57,14 @@ impl TimerTrigger { } /// Execute the first component in the application configuration. async fn handle(&self, msg: String) -> Result<()> { - let (mut store, instance) = self.engine.prepare_component( - &self.engine.config.components[0].id, - None, - None, - None, - None, - )?; + let (mut store, instance) = self + .engine + .prepare_component(&self.engine.config.components[0].id, None, None, None, None) + .await?; - let res = spawn_blocking(move || -> Result { - let t = spin_timer::SpinTimer::new(&mut store, &instance, |host| { - host.data.as_mut().unwrap() - })?; - Ok(t.handle_timer_request(&mut store, &msg)?) - }) - .await??; + let t = + spin_timer::SpinTimer::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?; + let res = t.handle_timer_request(&mut store, &msg).await?; log::info!("{}\n", res); Ok(()) From 840830abc9956ced32dcb4d1ca3236d6af806568 Mon Sep 17 00:00:00 2001 From: Vaughn Dice Date: Wed, 7 Sep 2022 11:56:51 -0600 Subject: [PATCH 012/163] ci(release.yml): create PR for template SDK updates Signed-off-by: Vaughn Dice --- .github/workflows/release.yml | 52 +++++++++++++++++++-------------- docs/content/release-process.md | 16 ++++++---- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b3132ddd6..8b2112851d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -221,7 +221,7 @@ jobs: steps: - uses: actions/checkout@v2 - - name: set the tag sdk/go/v* + - name: Set the tag to sdk/go/v* shell: bash run: echo "GO_SDK_TAG=sdk/go/${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV @@ -233,24 +233,18 @@ jobs: git tag ${{ env.GO_SDK_TAG }} git push origin ${{ env.GO_SDK_TAG }} - commit-and-create-template-tag: - name: Change sdk version in templates and create a new tag spin/templates/v* + create-template-sdk-update-pr: + name: Create PR with template SDK updates runs-on: ubuntu-latest needs: create-go-sdk-tag if: startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/checkout@v2 - - name: set the spin tag + - name: Set the spin tag shell: bash run: echo "SPIN_TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV - - name: set the tag spin/templates/v* - shell: bash - run: | - IFS=. read -r major minor patch <<< "${{ env.SPIN_TAG }}" - echo "TEMPLATE_TAG=spin/templates/$major.$minor" >> $GITHUB_ENV - - name: Change sdk version shell: bash run: | @@ -265,19 +259,33 @@ jobs: git_user_signingkey: true git_commit_gpgsign: true - - name: Commit changes - uses: stefanzweifel/git-auto-commit-action@v4 + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 with: - commit_message: "feat(templates): update sdk to ${{ env.SPIN_TAG }}" - commit_options: '-s -S' - commit_user_name: fermybot - commit_user_email: 103076628+fermybot@users.noreply.github.com - commit_author: fermybot <103076628+fermybot@users.noreply.github.com> - file_pattern: templates/* - branch: main - skip_dirty_check: true - skip_fetch: true - skip_checkout: true + commit-message: "feat(templates): update sdk to ${{ env.SPIN_TAG }}" + title: "feat(templates): update sdk to ${{ env.SPIN_TAG }}" + body: Update the SDK version used by the templates + branch: update-sdk-${{ env.SPIN_TAG }} + base: main + delete-branch: true + committer: fermybot <103076628+fermybot@users.noreply.github.com> + author: fermybot <103076628+fermybot@users.noreply.github.com> + signoff: true + + # This will run when the PR above is approved and merged into main via a merge commit + push-templates-tag: + runs-on: ubuntu-latest + needs: build + if: github.event.commits[0].author.name == 'fermybot' && contains(github.event.commits[0].message, 'update sdk') + steps: + - uses: actions/checkout@v2 + + - name: Set the tag to spin/templates/v* + shell: bash + run: | + spin_tag=$(echo "${{ github.event.commits[0].message }}" | grep -Eo v[0-9.]+) + IFS=. read -r major minor patch <<< "${spin_tag}" + echo "TEMPLATE_TAG=spin/templates/$major.$minor" >> $GITHUB_ENV - name: Tag spin/templates/v* and push it shell: bash diff --git a/docs/content/release-process.md b/docs/content/release-process.md index 9b3d555a38..5897cdaf19 100644 --- a/docs/content/release-process.md +++ b/docs/content/release-process.md @@ -17,13 +17,17 @@ To cut a release of Spin, you will need to do the following: 1. Before proceeding, verify that the merge commit on `main` intended to be tagged is green, i.e. CI is successful 1. Create a new tag with a `v` and then the version number (`v0.5.1`) -1. The Go SDK tag `sdk/go/v0.5.1` and template tag `spin/templates/v0.5` will - be created in the `release` - [action](https://github.com/fermyon/spin/actions/workflows/release.yaml). -1. When the `release` - [action](https://github.com/fermyon/spin/actions/workflows/release.yaml) - completes, binary artifacts and checksums will be automatically uploaded. +1. The Go SDK tag `sdk/go/v0.5.1` will be created in the [release action]. +1. When the [release action] completes, binary artifacts and checksums will be + automatically uploaded to the GitHub release. +1. A Pull Request will also be created by `fermybot` containing changes to the + templates per the updated SDK version. Once CI completes, approve this PR and + merge via a merge commit. This will trigger the `push-templates-tag` job in + the [release action], pushing the `spin/templates/v0.5` tag. (Note + that this tag may be force-pushed for all patch releases of a given minor release.) 1. Go to the GitHub [tags page](https://github.com/fermyon/spin/releases), edit a release, add the release notes. At this point, you can verify in the GitHub UI that the release was successful. + +[release action]: https://github.com/fermyon/spin/actions/workflows/release.yml \ No newline at end of file From 763387f890bb8adbd61ad881ae308f24a323c05b Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 8 Sep 2022 12:27:48 -0400 Subject: [PATCH 013/163] Simplify spin-config implementation - Replace the recursive tree structure with flat variables and per-component config maps. - Decouple config from spin-engine, turning it into a normal HostComponent. Signed-off-by: Lann Martin --- Cargo.lock | 5 +- crates/config/Cargo.toml | 3 + crates/config/src/host_component.rs | 53 +++-- crates/config/src/lib.rs | 325 +++++++++++++++------------- crates/config/src/provider.rs | 5 +- crates/config/src/provider/env.rs | 36 +-- crates/config/src/template.rs | 2 +- crates/config/src/tree.rs | 312 -------------------------- crates/engine/Cargo.toml | 3 +- crates/engine/src/lib.rs | 25 +-- crates/loader/src/bindle/config.rs | 6 +- crates/loader/src/bindle/mod.rs | 25 +-- crates/loader/src/common.rs | 33 +++ crates/loader/src/lib.rs | 1 + crates/loader/src/local/config.rs | 6 +- crates/loader/src/local/mod.rs | 25 +-- crates/manifest/Cargo.toml | 1 - crates/manifest/src/lib.rs | 17 +- crates/publish/src/expander.rs | 4 +- crates/testing/src/lib.rs | 4 +- crates/trigger/src/cli.rs | 38 +--- crates/trigger/src/lib.rs | 53 ++++- docs/content/configuration.md | 10 +- examples/spin-timer/src/main.rs | 1 + 24 files changed, 381 insertions(+), 612 deletions(-) delete mode 100644 crates/config/src/tree.rs create mode 100644 crates/loader/src/common.rs diff --git a/Cargo.lock b/Cargo.lock index 7f1463987c..acb3aea92e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3351,7 +3351,10 @@ name = "spin-config" version = "0.2.0" dependencies = [ "anyhow", + "async-trait", "serde", + "spin-engine", + "spin-manifest", "thiserror", "tokio", "toml", @@ -3367,7 +3370,6 @@ dependencies = [ "cap-std 0.24.4", "dirs 4.0.0", "sanitize-filename", - "spin-config", "spin-manifest", "tempfile", "tokio", @@ -3474,7 +3476,6 @@ dependencies = [ "anyhow", "indexmap", "serde", - "spin-config", "thiserror", "url", ] diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index a5f8405d64..a5e6211fa7 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -6,7 +6,10 @@ authors = [ "Fermyon Engineering " ] [dependencies] anyhow = "1.0" +async-trait = "0.1" serde = { version = "1.0", features = [ "derive" ] } +spin-engine = { path = "../engine" } +spin-manifest = { path = "../manifest" } thiserror = "1" tokio = { version = "1", features = [ "rt-multi-thread" ] } diff --git a/crates/config/src/host_component.rs b/crates/config/src/host_component.rs index 9d7f234211..1ac0a94e72 100644 --- a/crates/config/src/host_component.rs +++ b/crates/config/src/host_component.rs @@ -1,39 +1,56 @@ use std::sync::Arc; -use crate::{Error, Key, Resolver, TreePath}; +use spin_engine::host_component::HostComponent; +use spin_manifest::CoreComponent; +use wit_bindgen_wasmtime::async_trait; + +use crate::{Error, Key, Resolver}; mod wit { wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/spin-config.wit"], async: *}); } -pub use wit::spin_config::add_to_linker; -use wit_bindgen_wasmtime::async_trait; -/// A component configuration interface implementation. -pub struct ComponentConfig { - component_root: TreePath, +pub struct ConfigHostComponent { resolver: Arc, } -impl ComponentConfig { - pub fn new(component_id: impl Into, resolver: Arc) -> crate::Result { - let component_root = TreePath::new(component_id).or_else(|_| { - // Temporary mitigation for https://github.com/fermyon/spin/issues/337 - TreePath::new("invalid.path.issue_337") - })?; - Ok(Self { - component_root, - resolver, +impl ConfigHostComponent { + pub fn new(resolver: Resolver) -> Self { + Self { + resolver: Arc::new(resolver), + } + } +} + +impl HostComponent for ConfigHostComponent { + type State = ComponentConfig; + + fn add_to_linker( + linker: &mut wit_bindgen_wasmtime::wasmtime::Linker>, + state_handle: spin_engine::host_component::HostComponentsStateHandle, + ) -> anyhow::Result<()> { + wit::spin_config::add_to_linker(linker, move |ctx| state_handle.get_mut(ctx)) + } + + fn build_state(&self, component: &CoreComponent) -> anyhow::Result { + Ok(ComponentConfig { + component_id: component.id.clone(), + resolver: self.resolver.clone(), }) } } +/// A component configuration interface implementation. +pub struct ComponentConfig { + component_id: String, + resolver: Arc, +} + #[async_trait] impl wit::spin_config::SpinConfig for ComponentConfig { async fn get_config(&mut self, key: &str) -> Result { let key = Key::new(key)?; - let path = &self.component_root + key; - // TODO(lann): Make resolve async - tokio::task::block_in_place(|| Ok(self.resolver.resolve(&path)?)) + Ok(self.resolver.resolve(&self.component_id, key).await?) } } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 3df750684a..16a4faa541 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -2,127 +2,126 @@ pub mod host_component; pub mod provider; mod template; -mod tree; -use std::fmt::Debug; +use std::{borrow::Cow, collections::HashMap, fmt::Debug}; -pub use provider::Provider; -pub use tree::{Tree, TreePath}; +pub use async_trait::async_trait; +pub use provider::Provider; +use spin_manifest::Variable; use template::{Part, Template}; -/// A config resolution error. -#[derive(Debug, thiserror::Error)] -pub enum Error { - /// Invalid config key. - #[error("invalid config key: {0}")] - InvalidKey(String), - - /// Invalid config path. - #[error("invalid config path: {0}")] - InvalidPath(String), - - /// Invalid config schema. - #[error("invalid config schema: {0}")] - InvalidSchema(String), - - /// Invalid config template. - #[error("invalid config template: {0}")] - InvalidTemplate(String), - - /// Config provider error. - #[error("provider error: {0:?}")] - Provider(anyhow::Error), - - /// Unknown config path. - #[error("unknown config path: {0}")] - UnknownPath(String), -} - type Result = std::result::Result; /// A configuration resolver. #[derive(Debug, Default)] pub struct Resolver { - tree: Tree, + // variable key -> variable + variables: HashMap, + // component ID -> config key -> config value template + components_configs: HashMap>, providers: Vec>, } impl Resolver { /// Creates a Resolver for the given Tree. - pub fn new(tree: Tree) -> Result { + pub fn new(variables: impl IntoIterator) -> Result { + let variables: HashMap<_, _> = variables.into_iter().collect(); + // Validate keys so that we can rely on them during resolution + variables.keys().try_for_each(|key| Key::validate(key))?; Ok(Self { - tree, - providers: vec![], + variables, + components_configs: Default::default(), + providers: Default::default(), }) } + /// Adds component configuration values to the Resolver. + pub fn add_component_config( + &mut self, + component_id: impl Into, + config: impl IntoIterator, + ) -> Result<()> { + let component_id = component_id.into(); + let templates = config + .into_iter() + .map(|(key, val)| { + // Validate config keys so that we can rely on them during resolution + Key::validate(&key)?; + let template = self.validate_template(val)?; + Ok((key, template)) + }) + .collect::>()?; + + self.components_configs.insert(component_id, templates); + + Ok(()) + } + /// Adds a config Provider to the Resolver. pub fn add_provider(&mut self, provider: impl Provider + 'static) { self.providers.push(Box::new(provider)); } /// Resolves a config value for the given path. - pub fn resolve(&self, path: &TreePath) -> Result { - self.resolve_path(path, 0) + pub async fn resolve(&self, component_id: &str, key: Key<'_>) -> Result { + let configs = self.components_configs.get(component_id).ok_or_else(|| { + Error::UnknownPath(format!("no config for component {component_id:?}")) + })?; + + let key = key.as_ref(); + let template = configs + .get(key) + .ok_or_else(|| Error::UnknownPath(format!("no config for {component_id:?}.{key:?}")))?; + + self.resolve_template(template).await } - // Simple protection against infinite recursion - const RECURSION_LIMIT: usize = 100; - - // TODO(lann): make this non-recursive and/or "flatten" templates - fn resolve_path(&self, path: &TreePath, depth: usize) -> Result { - let depth = depth + 1; - if depth > Self::RECURSION_LIMIT { - return Err(Error::InvalidTemplate(format!( - "hit recursion limit at path: {}", - path - ))); + async fn resolve_template(&self, template: &Template) -> Result { + let mut resolved_parts: Vec> = Vec::with_capacity(template.parts().len()); + for part in template.parts() { + resolved_parts.push(match part { + Part::Lit(lit) => lit.as_ref().into(), + Part::Expr(var) => self.resolve_variable(var).await?.into(), + }); } - let slot = self.tree.get(path)?; - // If we're resolving top-level config we are ready to query provider(s). - if path.size() == 1 { - let key = path.keys().next().unwrap(); - for provider in &self.providers { - if let Some(value) = provider.get(&key).map_err(Error::Provider)? { - return Ok(value); - } + Ok(resolved_parts.concat()) + } + + async fn resolve_variable(&self, key: &str) -> Result { + let var = self + .variables + .get(key) + // This should have been caught by validate_template + .ok_or_else(|| Error::InvalidKey(key.to_string()))?; + + for provider in &self.providers { + if let Some(value) = provider.get(&Key(key)).await.map_err(Error::Provider)? { + return Ok(value); } } - // Resolve default template - if let Some(template) = &slot.default { - self.resolve_template(path, template, depth) - } else { - Err(Error::InvalidPath(format!( - "missing value at required path: {}", - path - ))) - } - } - fn resolve_template( - &self, - path: &TreePath, - template: &Template, - depth: usize, - ) -> Result { - template.parts().try_fold(String::new(), |value, part| { - Ok(match part { - Part::Lit(lit) => value + lit, - Part::Expr(expr) => { - let expr_path = if expr.starts_with('.') { - path.resolve_relative(expr)? - } else { - TreePath::new(expr.to_string())? - }; - value + &self.resolve_path(&expr_path, depth)? - } - }) + var.default.clone().ok_or_else(|| { + Error::Provider(anyhow::anyhow!( + "no provider resolved required variable {key:?}" + )) }) } + + fn validate_template(&self, template: String) -> Result