From 6769c8b5ad4fbc2f1734e6aad7e424e85ca622ce Mon Sep 17 00:00:00 2001 From: maciektr Date: Sat, 21 Dec 2024 00:41:39 +0100 Subject: [PATCH] Load prebuilt macros (#1856) Originally submitted as https://github.com/software-mansion/scarb/pull/1812 --------- Co-authored-by: Maksim Zdobnikau --- .github/workflows/ci.yml | 24 ++ Cargo.lock | 7 + Cargo.toml | 1 + scarb/Cargo.toml | 1 + .../bin/scarb/commands/proc_macro_server.rs | 18 +- scarb/src/compiler/compilation_unit.rs | 13 +- scarb/src/compiler/db.rs | 13 +- scarb/src/compiler/plugin/mod.rs | 7 +- .../compiler/plugin/proc_macro/compilation.rs | 25 ++ scarb/src/compiler/plugin/proc_macro/ffi.rs | 24 +- scarb/src/compiler/plugin/proc_macro/host.rs | 16 +- scarb/src/ops/compile.rs | 87 ++++--- scarb/src/ops/expand.rs | 13 +- scarb/src/ops/metadata.rs | 6 +- scarb/src/ops/resolve.rs | 140 ++++++++--- scarb/tests/build_cairo_plugin.rs | 6 +- scarb/tests/proc_macro_prebuilt.rs | 228 ++++++++++++++++++ .../src/cairo_plugin_project_builder.rs | 5 + .../src/proc_macro_server.rs | 25 ++ 19 files changed, 555 insertions(+), 104 deletions(-) create mode 100644 scarb/tests/proc_macro_prebuilt.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ed2eeab0..55f3e9bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,30 @@ jobs: - name: run tests run: cargo test -p scarb-metadata + test-prebuilt-plugins: + name: test prebuilt plugins ${{ matrix.platform.name }} + runs-on: ${{ matrix.platform.os }} + # This is isolated, so it can be run on more platforms. + strategy: + fail-fast: false + matrix: + platform: + - name: linux x86-64 + os: ubuntu-latest + - name: windows x86-64 + os: windows-latest + - name: macos arm64 + os: macos-latest + - name: macos x86-64 + os: macos-13 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Run prebuilt plugin tests + run: | + cargo test -p scarb --test proc_macro_prebuilt -- --ignored + check-rust: runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index 67f91547e..94f06621a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5029,6 +5029,7 @@ dependencies = [ "smol_str", "snapbox", "tar", + "target-triple", "test-case", "test-for-each-example", "thiserror 2.0.7", @@ -5826,6 +5827,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-triple" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a4d50cdb458045afc8131fd91b64904da29548bcb63c7236e0844936c13078" + [[package]] name = "tempfile" version = "3.12.0" diff --git a/Cargo.toml b/Cargo.toml index 69638e3bc..b9c9a045f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,6 +146,7 @@ xshell = "0.2" xxhash-rust = { version = "0.8", features = ["xxh3"] } zip = { version = "0.6", default-features = false, features = ["deflate"] } zstd = "0.13" +target-triple = "0.1" # Here we specify real dependency specifications for Cairo crates *if* currently we want to use # a particular unreleased commit (which is frequent mid-development). diff --git a/scarb/Cargo.toml b/scarb/Cargo.toml index e13b0f575..a045ebaae 100644 --- a/scarb/Cargo.toml +++ b/scarb/Cargo.toml @@ -90,6 +90,7 @@ windows-sys.workspace = true zstd.workspace = true cargo_metadata.workspace = true flate2.workspace = true +target-triple.workspace = true [target.'cfg(not(target_os = "linux"))'.dependencies] reqwest = { workspace = true, default-features = true } diff --git a/scarb/src/bin/scarb/commands/proc_macro_server.rs b/scarb/src/bin/scarb/commands/proc_macro_server.rs index 212a9faa1..73cd609ea 100644 --- a/scarb/src/bin/scarb/commands/proc_macro_server.rs +++ b/scarb/src/bin/scarb/commands/proc_macro_server.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use scarb::ops::CompilationUnitsOpts; use scarb::{ compiler::{plugin::proc_macro::ProcMacroHost, CairoCompilationUnit, CompilationUnit}, core::{Config, Workspace}, @@ -15,8 +16,11 @@ pub fn run(config: &mut Config) -> Result<()> { features: FeaturesSelector::AllFeatures, no_default_features: false, }, - true, &ws, + CompilationUnitsOpts { + ignore_cairo_version: true, + load_prebuilt_macros: true, + }, )?; // Compile procedural macros only. @@ -43,12 +47,12 @@ fn load_plugins( ws: &Workspace<'_>, proc_macros: &mut ProcMacroHost, ) -> Result<()> { - for plugin_info in unit - .cairo_plugins - .into_iter() - .filter(|plugin_info| !plugin_info.builtin) - { - proc_macros.register(plugin_info.package, ws.config())?; + for plugin_info in unit.cairo_plugins.into_iter().filter(|p| !p.builtin) { + if let Some(prebuilt) = plugin_info.prebuilt { + proc_macros.register_instance(prebuilt); + } else { + proc_macros.register_new(plugin_info.package, ws.config())?; + } } Ok(()) diff --git a/scarb/src/compiler/compilation_unit.rs b/scarb/src/compiler/compilation_unit.rs index cae3328d0..12758d409 100644 --- a/scarb/src/compiler/compilation_unit.rs +++ b/scarb/src/compiler/compilation_unit.rs @@ -1,14 +1,15 @@ -use std::fmt::Write; -use std::hash::{Hash, Hasher}; - use anyhow::{ensure, Result}; use cairo_lang_filesystem::cfg::CfgSet; use cairo_lang_filesystem::db::CrateIdentifier; use itertools::Itertools; use serde::{Deserialize, Serialize}; use smol_str::SmolStr; +use std::fmt::Write; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; use typed_builder::TypedBuilder; +use crate::compiler::plugin::proc_macro::ProcMacroInstance; use crate::compiler::Profile; use crate::core::{ ManifestCompilerConfig, Package, PackageId, PackageName, Target, TargetKind, Workspace, @@ -72,6 +73,9 @@ pub struct ProcMacroCompilationUnit { /// Rust compiler configuration parameters to use in this unit. pub compiler_config: serde_json::Value, + + /// Instance of the proc macro loaded from prebuilt library, if available. + pub prebuilt: Option>, } /// Information about a single package that is part of a [`CompilationUnit`]. @@ -96,8 +100,11 @@ pub struct CompilationUnitComponent { pub struct CompilationUnitCairoPlugin { /// The Scarb plugin [`Package`] to load. pub package: Package, + /// Indicate whether the plugin is built into Scarb, or compiled from source. pub builtin: bool, pub prebuilt_allowed: bool, + /// Instance of the proc macro loaded from prebuilt library, if available. + pub prebuilt: Option>, } /// Unique identifier of the compilation unit component. diff --git a/scarb/src/compiler/db.rs b/scarb/src/compiler/db.rs index 429dd5813..66fad856d 100644 --- a/scarb/src/compiler/db.rs +++ b/scarb/src/compiler/db.rs @@ -1,3 +1,7 @@ +use crate::compiler::plugin::proc_macro::{ProcMacroHost, ProcMacroHostPlugin}; +use crate::compiler::{CairoCompilationUnit, CompilationUnitAttributes, CompilationUnitComponent}; +use crate::core::Workspace; +use crate::DEFAULT_MODULE_MAIN_FILE; use anyhow::{anyhow, Result}; use cairo_lang_compiler::db::{RootDatabase, RootDatabaseBuilder}; use cairo_lang_compiler::project::{AllCratesConfig, ProjectConfig, ProjectConfigContent}; @@ -14,11 +18,6 @@ use std::path::PathBuf; use std::sync::Arc; use tracing::trace; -use crate::compiler::plugin::proc_macro::{ProcMacroHost, ProcMacroHostPlugin}; -use crate::compiler::{CairoCompilationUnit, CompilationUnitAttributes, CompilationUnitComponent}; -use crate::core::Workspace; -use crate::DEFAULT_MODULE_MAIN_FILE; - pub struct ScarbDatabase { pub db: RootDatabase, pub proc_macro_host: Arc, @@ -59,8 +58,10 @@ fn load_plugins( let plugin = ws.config().cairo_plugins().fetch(package_id)?; let instance = plugin.instantiate()?; builder.with_plugin_suite(instance.plugin_suite()); + } else if let Some(prebuilt) = &plugin_info.prebuilt { + proc_macros.register_instance(prebuilt.clone()); } else { - proc_macros.register(plugin_info.package.clone(), ws.config())?; + proc_macros.register_new(plugin_info.package.clone(), ws.config())?; } } let macro_host = Arc::new(proc_macros.into_plugin()?); diff --git a/scarb/src/compiler/plugin/mod.rs b/scarb/src/compiler/plugin/mod.rs index 86f6d153d..3f9b15475 100644 --- a/scarb/src/compiler/plugin/mod.rs +++ b/scarb/src/compiler/plugin/mod.rs @@ -9,6 +9,7 @@ use itertools::Itertools; use serde::{Deserialize, Serialize}; use crate::compiler::plugin::builtin::BuiltinCairoRunPlugin; +use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider; use crate::core::{Package, PackageId, TargetKind, Workspace}; use self::builtin::{BuiltinStarkNetPlugin, BuiltinTestPlugin}; @@ -29,8 +30,10 @@ pub fn fetch_cairo_plugin(package: &Package, ws: &Workspace<'_>) -> Result<()> { assert!(package.is_cairo_plugin()); let target = package.fetch_target(&TargetKind::CAIRO_PLUGIN)?; let props: CairoPluginProps = target.props()?; - // No need to fetch for buildin plugins. - if !props.builtin { + // There is no need to run `cargo fetch` for builtin plugins. + // The `fetch` will not be run for a proc macro that contains a prebuilt library file. + // Note, that in case the prebuilt lib file is corrupted, it will be later compiled with Cargo anyway. + if !props.builtin && package.prebuilt_lib_path().is_none() { proc_macro::fetch_crate(package, ws)?; } Ok(()) diff --git a/scarb/src/compiler/plugin/proc_macro/compilation.rs b/scarb/src/compiler/plugin/proc_macro/compilation.rs index cebcb2949..684353c25 100644 --- a/scarb/src/compiler/plugin/proc_macro/compilation.rs +++ b/scarb/src/compiler/plugin/proc_macro/compilation.rs @@ -15,12 +15,14 @@ use ra_ap_toolchain::Tool; use scarb_ui::{Message, OutputFormat}; use serde::{Serialize, Serializer}; use serde_json::value::RawValue; +use std::env::consts::DLL_SUFFIX; use std::fmt::Display; use std::fs; use std::io::{Seek, SeekFrom}; use std::ops::Deref; use std::process::Command; use tar::Archive; +use target_triple::target; use tracing::trace_span; pub const PROC_MACRO_BUILD_PROFILE: &str = "release"; @@ -31,6 +33,8 @@ pub trait SharedLibraryProvider { fn target_path(&self, config: &Config) -> Filesystem; /// Location of the shared library for the package. fn shared_lib_path(&self, config: &Config) -> Result; + /// Location of the prebuilt binary for the package, if defined. + fn prebuilt_lib_path(&self) -> Option; } impl SharedLibraryProvider for Package { @@ -61,6 +65,27 @@ impl SharedLibraryProvider for Package { .path_unchecked() .join(lib_name)) } + + fn prebuilt_lib_path(&self) -> Option { + let target_triple = target!(); + + let prebuilt_name = format!( + "{name}_v{version}_{target}{suffix}", + name = self.id.name, + version = self.id.version, + target = target_triple, + suffix = DLL_SUFFIX + ); + + let prebuilt_path = self + .root() + .join("target") + .join("scarb") + .join("cairo-plugin") + .join(prebuilt_name); + + prebuilt_path.exists().then_some(prebuilt_path) + } } pub fn compile_unit(unit: ProcMacroCompilationUnit, ws: &Workspace<'_>) -> Result<()> { diff --git a/scarb/src/compiler/plugin/proc_macro/ffi.rs b/scarb/src/compiler/plugin/proc_macro/ffi.rs index 4d42bab6c..2e77441ee 100644 --- a/scarb/src/compiler/plugin/proc_macro/ffi.rs +++ b/scarb/src/compiler/plugin/proc_macro/ffi.rs @@ -1,4 +1,4 @@ -use crate::core::{Config, Package, PackageId}; +use crate::core::{Package, PackageId}; use anyhow::{ensure, Context, Result}; use cairo_lang_defs::patcher::PatchBuilder; use cairo_lang_macro::{ @@ -26,6 +26,7 @@ use libloading::os::unix::Symbol as RawSymbol; #[cfg(windows)] use libloading::os::windows::Symbol as RawSymbol; use smol_str::SmolStr; +use tracing::trace; pub trait FromSyntaxNode { fn from_syntax_node(db: &dyn SyntaxGroup, node: &impl TypedSyntaxNode) -> Self; @@ -61,11 +62,22 @@ impl Debug for ProcMacroInstance { impl ProcMacroInstance { /// Load shared library - pub fn try_new(package: Package, config: &Config) -> Result { - let lib_path = package - .shared_lib_path(config) - .context("could not resolve shared library path")?; - let plugin = unsafe { Plugin::try_new(lib_path.to_path_buf())? }; + pub fn try_new(package_id: PackageId, lib_path: Utf8PathBuf) -> Result { + trace!("loading compiled macro for `{}` package", package_id); + let plugin = unsafe { Plugin::try_new(lib_path)? }; + Ok(Self { + expansions: unsafe { Self::load_expansions(&plugin, package_id)? }, + package_id, + plugin, + }) + } + + pub fn try_load_prebuilt(package: Package) -> Result { + trace!("loading prebuilt macro for `{}` package", package.id); + let prebuilt_path = package + .prebuilt_lib_path() + .context("could not resolve prebuilt library path")?; + let plugin = unsafe { Plugin::try_new(prebuilt_path)? }; Ok(Self { expansions: unsafe { Self::load_expansions(&plugin, package.id)? }, package_id: package.id, diff --git a/scarb/src/compiler/plugin/proc_macro/host.rs b/scarb/src/compiler/plugin/proc_macro/host.rs index 0886abb81..7e8765ef9 100644 --- a/scarb/src/compiler/plugin/proc_macro/host.rs +++ b/scarb/src/compiler/plugin/proc_macro/host.rs @@ -1,8 +1,9 @@ +use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider; use crate::compiler::plugin::proc_macro::{ Expansion, ExpansionKind, FromSyntaxNode, ProcMacroInstance, }; use crate::core::{Config, Package, PackageId}; -use anyhow::{ensure, Result}; +use anyhow::{ensure, Context, Result}; use cairo_lang_defs::ids::{ModuleItemId, TopLevelLanguageElementId}; use cairo_lang_defs::patcher::{PatchBuilder, RewriteNode}; use cairo_lang_defs::plugin::{ @@ -1150,9 +1151,16 @@ pub struct ProcMacroHost { } impl ProcMacroHost { - pub fn register(&mut self, package: Package, config: &Config) -> Result<()> { - let instance = ProcMacroInstance::try_new(package, config)?; - self.macros.push(Arc::new(instance)); + pub fn register_instance(&mut self, instance: Arc) { + self.macros.push(instance); + } + + pub fn register_new(&mut self, package: Package, config: &Config) -> Result<()> { + let lib_path = package + .shared_lib_path(config) + .context("could not resolve shared library path")?; + let instance = ProcMacroInstance::try_new(package.id, lib_path)?; + self.register_instance(Arc::new(instance)); Ok(()) } diff --git a/scarb/src/ops/compile.rs b/scarb/src/ops/compile.rs index 3e63617e8..565cfbb09 100644 --- a/scarb/src/ops/compile.rs +++ b/scarb/src/ops/compile.rs @@ -19,7 +19,7 @@ use crate::core::{ FeatureName, PackageId, PackageName, TargetKind, Utf8PathWorkspaceExt, Workspace, }; use crate::ops; -use crate::ops::{get_test_package_ids, validate_features}; +use crate::ops::{get_test_package_ids, validate_features, CompilationUnitsOpts}; #[derive(Debug, Clone)] pub enum FeaturesSelector { @@ -125,36 +125,43 @@ where validate_features(&packages_to_process, &opts.features)?; // Add test compilation units to build let packages = get_test_package_ids(packages, ws); - let compilation_units = - ops::generate_compilation_units(&resolve, &opts.features, opts.ignore_cairo_version, ws)? - .into_iter() - .filter(|cu| { - let is_excluded = opts - .exclude_target_kinds - .contains(&cu.main_component().target_kind()); - let is_included = opts.include_target_kinds.is_empty() - || opts - .include_target_kinds - .contains(&cu.main_component().target_kind()); - let is_included = is_included - && (opts.include_target_names.is_empty() - || cu - .main_component() - .targets - .iter() - .any(|t| opts.include_target_names.contains(&t.name))); - let is_selected = packages.contains(&cu.main_package_id()); - let is_cairo_plugin = matches!(cu, CompilationUnit::ProcMacro(_)); - is_cairo_plugin || (is_selected && is_included && !is_excluded) - }) - .sorted_by_key(|cu| { - if matches!(cu, CompilationUnit::ProcMacro(_)) { - 0 - } else { - 1 - } - }) - .collect::>(); + let compilation_units = ops::generate_compilation_units( + &resolve, + &opts.features, + ws, + CompilationUnitsOpts { + ignore_cairo_version: opts.ignore_cairo_version, + load_prebuilt_macros: true, + }, + )? + .into_iter() + .filter(|cu| { + let is_excluded = opts + .exclude_target_kinds + .contains(&cu.main_component().target_kind()); + let is_included = opts.include_target_kinds.is_empty() + || opts + .include_target_kinds + .contains(&cu.main_component().target_kind()); + let is_included = is_included + && (opts.include_target_names.is_empty() + || cu + .main_component() + .targets + .iter() + .any(|t| opts.include_target_names.contains(&t.name))); + let is_selected = packages.contains(&cu.main_package_id()); + let is_cairo_plugin = matches!(cu, CompilationUnit::ProcMacro(_)); + is_cairo_plugin || (is_selected && is_included && !is_excluded) + }) + .sorted_by_key(|cu| { + if matches!(cu, CompilationUnit::ProcMacro(_)) { + 0 + } else { + 1 + } + }) + .collect::>(); operation(compilation_units, ws)?; @@ -194,13 +201,21 @@ pub fn compile_unit(unit: CompilationUnit, ws: &Workspace<'_>) -> Result<()> { fn compile_unit_inner(unit: CompilationUnit, ws: &Workspace<'_>) -> Result<()> { let package_name = unit.main_package_id().name.clone(); - ws.config() - .ui() - .print(Status::new("Compiling", &unit.name())); - let result = match unit { - CompilationUnit::ProcMacro(unit) => proc_macro::compile_unit(unit, ws), + CompilationUnit::ProcMacro(unit) => { + if unit.prebuilt.is_some() { + Ok(()) + } else { + ws.config() + .ui() + .print(Status::new("Compiling", &unit.name())); + proc_macro::compile_unit(unit, ws) + } + } CompilationUnit::Cairo(unit) => { + ws.config() + .ui() + .print(Status::new("Compiling", &unit.name())); let ScarbDatabase { mut db, proc_macro_host, diff --git a/scarb/src/ops/expand.rs b/scarb/src/ops/expand.rs index 417fa4b1b..e5a47f218 100644 --- a/scarb/src/ops/expand.rs +++ b/scarb/src/ops/expand.rs @@ -3,7 +3,7 @@ use crate::compiler::helpers::{build_compiler_config, write_string}; use crate::compiler::{CairoCompilationUnit, CompilationUnit, CompilationUnitAttributes}; use crate::core::{Package, PackageId, TargetKind, Workspace}; use crate::ops; -use crate::ops::{get_test_package_ids, validate_features, FeaturesOpts}; +use crate::ops::{get_test_package_ids, validate_features, CompilationUnitsOpts, FeaturesOpts}; use anyhow::{bail, Context, Result}; use cairo_lang_defs::db::DefsGroup; use cairo_lang_defs::ids::{LanguageElementId, ModuleId, ModuleItemId}; @@ -43,8 +43,15 @@ pub fn expand(package: Package, opts: ExpandOpts, ws: &Workspace<'_>) -> Result< let package_name = package.id.name.to_string(); let resolve = ops::resolve_workspace(ws)?; - let compilation_units = - ops::generate_compilation_units(&resolve, &opts.features, opts.ignore_cairo_version, ws)?; + let compilation_units = ops::generate_compilation_units( + &resolve, + &opts.features, + ws, + CompilationUnitsOpts { + ignore_cairo_version: opts.ignore_cairo_version, + load_prebuilt_macros: true, + }, + )?; // Compile procedural macros. compilation_units diff --git a/scarb/src/ops/metadata.rs b/scarb/src/ops/metadata.rs index 4429dec6d..f4d4ccc43 100644 --- a/scarb/src/ops/metadata.rs +++ b/scarb/src/ops/metadata.rs @@ -17,6 +17,7 @@ use crate::core::{ SourceId, Target, Workspace, }; use crate::ops; +use crate::ops::CompilationUnitsOpts; use crate::version::CommitInfo; pub struct MetadataOptions { @@ -47,8 +48,11 @@ pub fn collect_metadata(opts: &MetadataOptions, ws: &Workspace<'_>) -> Result = ops::generate_compilation_units( &resolve, &opts.features, - opts.ignore_cairo_version, ws, + CompilationUnitsOpts { + ignore_cairo_version: opts.ignore_cairo_version, + load_prebuilt_macros: false, + }, )? .iter() .flat_map(collect_compilation_unit_metadata) diff --git a/scarb/src/ops/resolve.rs b/scarb/src/ops/resolve.rs index ae2cd9cbb..177d9f919 100644 --- a/scarb/src/ops/resolve.rs +++ b/scarb/src/ops/resolve.rs @@ -1,3 +1,5 @@ +use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider; +use crate::compiler::plugin::proc_macro::ProcMacroInstance; use crate::compiler::plugin::{fetch_cairo_plugin, CairoPluginProps}; use crate::compiler::{ CairoCompilationUnit, CompilationUnit, CompilationUnitAttributes, CompilationUnitCairoPlugin, @@ -27,6 +29,7 @@ use indoc::formatdoc; use itertools::Itertools; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::iter::zip; +use std::sync::Arc; pub struct WorkspaceResolve { pub resolve: Resolve, @@ -225,45 +228,98 @@ impl AllowedPrebuiltFilter { } } +pub struct CompilationUnitsOpts { + /// Skip cairo version requirements check. + /// + /// This will ignore requirements defined in `cairo-version` field in package manifest. + pub ignore_cairo_version: bool, + /// Load prebuilt macros for procedural macros. + /// + /// Disabling this flag is useful if the generated compilation units will not be used to compile + /// users project. For example, when generating units for scarb-metadata. + /// Note, even if `true`, only macros allowed in package manifest will be loaded. + pub load_prebuilt_macros: bool, +} + #[tracing::instrument(skip_all, level = "debug")] pub fn generate_compilation_units( resolve: &WorkspaceResolve, enabled_features: &FeaturesOpts, - ignore_cairo_version: bool, ws: &Workspace<'_>, + opts: CompilationUnitsOpts, ) -> Result> { - let mut units = Vec::with_capacity(ws.members().size_hint().0); + let mut cairo_units: Vec = Vec::with_capacity(ws.members().size_hint().0); let members = ws .members() .filter(|member| !member.is_cairo_plugin()) .collect_vec(); validate_features(&members, enabled_features)?; for member in members { - units.extend(generate_cairo_compilation_units( + cairo_units.extend(generate_cairo_compilation_units( &member, resolve, enabled_features, - ignore_cairo_version, + opts.ignore_cairo_version, ws, )?); } - let cairo_plugins = units + let proc_macro_units = cairo_units .iter() - .filter_map(|unit| match unit { - CompilationUnit::Cairo(unit) => Some(unit), - _ => None, - }) .flat_map(|unit| unit.cairo_plugins.clone()) .filter(|plugin| !plugin.builtin) - .map(|plugin| plugin.package.clone()) - .chain(ws.members().filter(|member| member.is_cairo_plugin())) - .unique_by(|plugin| plugin.id) - .collect_vec(); + .map(|plugin| (plugin.package.clone(), plugin.prebuilt_allowed)) + .chain( + ws.members() + .filter(|member| member.is_cairo_plugin()) + .map(|member| (member, false)), + ) + // In case some prebuilt macro is allowed for one workspace member and disallowed for + // the other, we need to set `prebuilt_allowed` to `false` for that macro package, so that + // it is compiled with Cargo. + // This works by placing packages with `prebuilt_allowed` set to `false` first + // in the iterator and relying on stability of `unique_by` method to skip duplicates with + // allowed prebuilt macros (retaining once with disabled). + .sorted_by_key(|(_, prebuilt_allowed)| if *prebuilt_allowed { 1 } else { 0 }) + .unique_by(|(plugin, _)| plugin.id) + .map(|(plugin, prebuilt_allowed)| { + Ok(( + plugin.id, + generate_cairo_plugin_compilation_units( + &plugin, + opts.load_prebuilt_macros && prebuilt_allowed, + )?, + )) + }) + .collect::>>()?; - for plugin in cairo_plugins { - units.extend(generate_cairo_plugin_compilation_units(&plugin)?); - } + let units = cairo_units + .into_iter() + .map(|mut unit| { + for plugin in &mut unit.cairo_plugins { + if let Some(proc_macro_unit) = proc_macro_units.get(&plugin.package.id) { + plugin.prebuilt = plugin + // We check if prebuilt is allowed for this compilation unit, as it might + // be disabled for some workspace members even if other members allow it. + .prebuilt_allowed + .then_some(proc_macro_unit.prebuilt.clone()) + .flatten(); + } + } + unit + }) + .map(CompilationUnit::Cairo) + .collect_vec(); + let units: Vec = units + .into_iter() + .chain( + proc_macro_units + .into_values() + // Sort for stability. + .sorted_by_key(|unit| unit.main_package_id) + .map(CompilationUnit::ProcMacro), + ) + .collect(); assert!( units.iter().map(CompilationUnit::id).all_unique(), @@ -298,7 +354,7 @@ fn generate_cairo_compilation_units( enabled_features: &FeaturesOpts, ignore_cairo_version: bool, ws: &Workspace<'_>, -) -> Result> { +) -> Result> { let profile = ws.current_profile()?; let mut solution = PackageSolutionCollector::new(member, resolve, ws); let grouped = member @@ -312,14 +368,14 @@ fn generate_cairo_compilation_units( .sorted_by_key(|(_, group)| group[0].kind.clone()) .map(|(_group_id, group)| { let group = group.into_iter().cloned().collect_vec(); - Ok(CompilationUnit::Cairo(cairo_compilation_unit_for_target( + cairo_compilation_unit_for_target( group, member, profile.clone(), enabled_features, ignore_cairo_version, &mut solution, - )?)) + ) }) .collect::>>()?; let result = member @@ -328,14 +384,14 @@ fn generate_cairo_compilation_units( .iter() .filter(|target| target.group_id.is_none()) .map(|member_target| { - Ok(CompilationUnit::Cairo(cairo_compilation_unit_for_target( + cairo_compilation_unit_for_target( vec![member_target.clone()], member, profile.clone(), enabled_features, ignore_cairo_version, &mut solution, - )?)) + ) }) .collect::>>()? .into_iter() @@ -653,6 +709,7 @@ impl<'a> PackageSolutionCollector<'a> { .package(package) .builtin(props.builtin) .prebuilt_allowed(prebuilt_allowed) + .prebuilt(None) .build()) }) .collect::>>()?; @@ -754,20 +811,37 @@ fn check_cairo_version_compatibility( Ok(()) } -fn generate_cairo_plugin_compilation_units(member: &Package) -> Result> { - Ok(vec![CompilationUnit::ProcMacro(ProcMacroCompilationUnit { +pub fn generate_cairo_plugin_compilation_units( + member: &Package, + // Whether loading a prebuilt library is both allowed and requested. + load_prebuilt: bool, +) -> Result { + let load_prebuilt = load_prebuilt && member.prebuilt_lib_path().is_some(); + let prebuilt = load_prebuilt + .then_some( + ProcMacroInstance::try_load_prebuilt(member.clone()) + // Note we ignore loading errors here. + // If the prebuilt library is corrupted, it will be later compiled with Cargo, + // like there was no prebuilt defined. + .ok() + .map(Arc::new), + ) + .flatten(); + let components = vec![CompilationUnitComponent::try_new( + member.clone(), + vec![member + .fetch_target(&TargetKind::CAIRO_PLUGIN) + .cloned() + // Safe to unwrap, as member.is_cairo_plugin() has been ensured before. + .expect("main component of procedural macro must define `cairo-plugin` target")], + None, + )?]; + Ok(ProcMacroCompilationUnit { main_package_id: member.id, compiler_config: serde_json::Value::Null, - components: vec![CompilationUnitComponent::try_new( - member.clone(), - vec![member - .fetch_target(&TargetKind::CAIRO_PLUGIN) - .cloned() - // Safe to unwrap, as member.is_cairo_plugin() has been ensured before. - .expect("main component of procedural macro must define `cairo-plugin` target")], - None, - )?], - })]) + components, + prebuilt, + }) } /// Generate package ids associated with test compilation units for the given packages. diff --git a/scarb/tests/build_cairo_plugin.rs b/scarb/tests/build_cairo_plugin.rs index c93fb3b38..042b685cf 100644 --- a/scarb/tests/build_cairo_plugin.rs +++ b/scarb/tests/build_cairo_plugin.rs @@ -96,8 +96,8 @@ fn can_check_cairo_project_with_plugins() { .assert() .success() .stdout_matches(indoc! {r#" - [..]Compiling some v1.0.0 ([..]Scarb.toml) [..]Checking other v1.0.0 ([..]Scarb.toml) + [..]Compiling some v1.0.0 ([..]Scarb.toml) [..]Checking hello v1.0.0 ([..]Scarb.toml) [..]Finished checking `dev` profile target(s) in [..] "#}); @@ -649,8 +649,8 @@ fn can_define_multiple_macros() { .assert() .success() .stdout_matches(indoc! {r#" - [..]Compiling some v1.0.0 ([..]Scarb.toml) [..]Compiling other v1.0.0 ([..]Scarb.toml) + [..]Compiling some v1.0.0 ([..]Scarb.toml) [..]Compiling hello v1.0.0 ([..]Scarb.toml) [..]Finished `dev` profile target(s) in [..] [..]Running hello @@ -753,8 +753,8 @@ fn cannot_duplicate_macros_across_packages() { .assert() .failure() .stdout_matches(indoc! {r#" - [..]Compiling some v1.0.0 ([..]Scarb.toml) [..]Compiling other v1.0.0 ([..]Scarb.toml) + [..]Compiling some v1.0.0 ([..]Scarb.toml) [..]Compiling hello v1.0.0 ([..]Scarb.toml) error: duplicate expansions defined for procedural macros: hello (some v1.0.0 ([..]Scarb.toml) and other v1.0.0 ([..]Scarb.toml)) "#}); diff --git a/scarb/tests/proc_macro_prebuilt.rs b/scarb/tests/proc_macro_prebuilt.rs new file mode 100644 index 000000000..d58de21b3 --- /dev/null +++ b/scarb/tests/proc_macro_prebuilt.rs @@ -0,0 +1,228 @@ +use assert_fs::fixture::{ChildPath, FileWriteStr, PathCreateDir}; +use assert_fs::prelude::PathChild; +use assert_fs::TempDir; +use cairo_lang_macro::TokenStream; +use indoc::indoc; +use libloading::library_filename; +use scarb_proc_macro_server_types::methods::expand::{ExpandInline, ExpandInlineMacroParams}; +use scarb_test_support::cairo_plugin_project_builder::CairoPluginProjectBuilder; +use scarb_test_support::command::Scarb; +use scarb_test_support::proc_macro_server::ProcMacroClient; +use scarb_test_support::project_builder::ProjectBuilder; +use scarb_test_support::workspace_builder::WorkspaceBuilder; +use snapbox::cmd::Command; +use std::fs; + +static TRIPLETS: [(&str, &str); 4] = [ + ("aarch64-apple-darwin", ".dylib"), + ("x86_64-apple-darwin", ".dylib"), + ("x86_64-unknown-linux-gnu", ".so"), + ("x86_64-pc-windows-msvc", ".dll"), +]; + +fn proc_macro_example(t: &ChildPath) { + let name = "proc_macro_example"; + let version = "0.1.0"; + CairoPluginProjectBuilder::default() + .name(name) + .version(version) + .lib_rs(indoc! {r#" + use cairo_lang_macro::{ProcMacroResult, TokenStream, inline_macro}; + #[inline_macro] + pub fn some(token_stream: TokenStream) -> ProcMacroResult { + ProcMacroResult::new(token_stream) + } + "#}) + .build(t); + let dll_filename = library_filename(name); + let dll_filename = dll_filename.to_string_lossy().to_string(); + let build_dir = t.child("cargo_build_dir"); + Command::new("cargo") + .arg("build") + .arg("--release") + .env("CARGO_TARGET_DIR", build_dir.path()) + .current_dir(t) + .assert() + .success(); + t.child("target/scarb/cairo-plugin") + .create_dir_all() + .unwrap(); + for (target, extension) in TRIPLETS { + let target_name = format!("{name}_v{version}_{target}{extension}"); + fs::copy( + build_dir.child("release").child(dll_filename.clone()), + t.child("target/scarb/cairo-plugin/").child(target_name), + ) + .unwrap(); + } +} + +#[test] +#[ignore = "run this test by name"] +fn compile_with_prebuilt_plugins() { + let t = TempDir::new().unwrap(); + proc_macro_example(&t.child("dep")); + let builder = |name: &str| { + ProjectBuilder::start() + .name(name) + .lib_cairo(indoc! {r#" + fn main() -> u32 { + let x = some!(42); + x + } + "#}) + .dep("proc_macro_example", t.child("dep")) + .manifest_extra(indoc! {r#" + [tool.scarb] + allow-prebuilt-plugins = ["proc_macro_example"] + "#}) + }; + builder("a").build(&t.child("a")); + builder("b").build(&t.child("b")); + WorkspaceBuilder::start() + .add_member("a") + .add_member("b") + .build(&t); + Scarb::quick_snapbox() + .arg("build") + // Disable Cargo and Rust compiler. + .env("CARGO", "/bin/false") + .env("RUSTC", "/bin/false") + .current_dir(&t) + .assert() + .success() + .stdout_matches(indoc! {r#" + [..]Compiling a v1.0.0 ([..]Scarb.toml) + [..]Compiling b v1.0.0 ([..]Scarb.toml) + [..] Finished `dev` profile target(s) in [..] + "#}); +} + +#[test] +#[ignore = "run this test by name"] +fn compile_with_prebuilt_plugins_only_one_allows() { + let t = TempDir::new().unwrap(); + proc_macro_example(&t.child("dep")); + let builder = |name: &str, allow: bool| { + let b = ProjectBuilder::start() + .name(name) + .lib_cairo(indoc! {r#" + fn main() -> u32 { + let x = some!(42); + x + } + "#}) + .dep("proc_macro_example", t.child("dep")); + if allow { + b.manifest_extra(indoc! {r#" + [tool.scarb] + allow-prebuilt-plugins = ["proc_macro_example"] + "#}) + } else { + b + } + }; + builder("a", true).build(&t.child("a")); + builder("b", false).build(&t.child("b")); + WorkspaceBuilder::start() + .add_member("a") + .add_member("b") + .build(&t); + Scarb::quick_snapbox() + .arg("build") + .current_dir(&t) + // Disable output from Cargo. + .env("CARGO_TERM_QUIET", "true") + .assert() + .success() + .stdout_matches(indoc! {r#" + [..]Compiling proc_macro_example v0.1.0 ([..]) + [..]Compiling a v1.0.0 ([..]Scarb.toml) + [..]Compiling b v1.0.0 ([..]Scarb.toml) + [..] Finished `dev` profile target(s) in [..] + "#}); +} + +fn invalid_prebuilt_project(t: &ChildPath) { + let name = "invalid_prebuilt_example"; + let version = "0.1.0"; + CairoPluginProjectBuilder::default() + .name(name) + .version(version) + .lib_rs(indoc! {r#" + use cairo_lang_macro::{ProcMacroResult, TokenStream, inline_macro}; + #[inline_macro] + pub fn some(token_stream: TokenStream) -> ProcMacroResult { + ProcMacroResult::new(token_stream) + } + "#}) + .build(t); + let target = t.child("target/scarb/cairo-plugin"); + for (triplet, extension) in TRIPLETS { + let path = format!("{name}_v{version}_{triplet}{extension}"); + target + .child(path) + .write_str("this is not a valid lib") + .unwrap(); + } +} + +#[test] +#[ignore = "run this test by name"] +fn compile_with_invalid_prebuilt_plugins() { + let t = TempDir::new().unwrap(); + invalid_prebuilt_project(&t.child("dep")); + ProjectBuilder::start() + .name("hello") + .lib_cairo(indoc! {r#" + fn main() -> u32 { + let x = some!(42); + x + } + "#}) + .dep("invalid_prebuilt_example", t.child("dep")) + .manifest_extra(indoc! {r#" + [tool.scarb] + allow-prebuilt-plugins = ["invalid_prebuilt_example"] + "#}) + .build(&t); + Scarb::quick_snapbox() + .arg("build") + // Disable output from Cargo. + .env("CARGO_TERM_QUIET", "true") + .current_dir(&t) + .assert() + .success() + .stdout_matches(indoc! {r#" + [..]Compiling invalid_prebuilt_example v0.1.0 ([..]) + [..]Compiling hello v1.0.0 ([..]Scarb.toml) + [..] Finished `dev` profile target(s) in [..] + "#}); +} + +#[test] +#[ignore = "run this test by name"] +fn load_prebuilt_proc_macros() { + let t = TempDir::new().unwrap(); + proc_macro_example(&t.child("dep")); + let project = t.child("test_package"); + ProjectBuilder::start() + .name("test_package") + .version("1.0.0") + .lib_cairo("") + .dep("proc_macro_example", t.child("dep")) + .manifest_extra(indoc! {r#" + [tool.scarb] + allow-prebuilt-plugins = ["proc_macro_example"] + "#}) + .build(&project); + let mut proc_macro_server = ProcMacroClient::new_without_cargo(&project); + let response = proc_macro_server + .request_and_wait::(ExpandInlineMacroParams { + name: "some".to_string(), + args: TokenStream::new("42".to_string()), + }) + .unwrap(); + assert_eq!(response.diagnostics, vec![]); + assert_eq!(response.token_stream, TokenStream::new("42".to_string())); +} diff --git a/utils/scarb-test-support/src/cairo_plugin_project_builder.rs b/utils/scarb-test-support/src/cairo_plugin_project_builder.rs index bc8fae976..f869b5ff0 100644 --- a/utils/scarb-test-support/src/cairo_plugin_project_builder.rs +++ b/utils/scarb-test-support/src/cairo_plugin_project_builder.rs @@ -45,6 +45,11 @@ impl CairoPluginProjectBuilder { self } + pub fn version(mut self, version: impl ToString) -> Self { + self.project = self.project.version(&version.to_string()); + self + } + pub fn src(mut self, path: impl Into, source: impl ToString) -> Self { self.src.insert(path.into(), source.to_string()); self diff --git a/utils/scarb-test-support/src/proc_macro_server.rs b/utils/scarb-test-support/src/proc_macro_server.rs index 12e93176f..cf1fefab0 100644 --- a/utils/scarb-test-support/src/proc_macro_server.rs +++ b/utils/scarb-test-support/src/proc_macro_server.rs @@ -90,6 +90,31 @@ impl ProcMacroClient { responses: Default::default(), } } + pub fn new_without_cargo>(path: P) -> Self { + let mut server_process = Scarb::new() + .std() + .arg("--quiet") + .arg("proc-macro-server") + .env("CARGO", "/bin/false") + .env("RUSTC", "/bin/false") + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .stderr(Stdio::inherit()) + .current_dir(path) + .spawn() + .unwrap(); + + let requester = server_process.stdin.take().unwrap(); + let responder = BufReader::new(server_process.stdout.take().unwrap()).lines(); + + Self { + requester, + responder, + server_process, + id_counter: Default::default(), + responses: Default::default(), + } + } pub fn request(&mut self, params: M::Params) -> PendingRequest { let id = self.id_counter;