diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..3550a30f2 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index 51de44bf3..e971a4539 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ working_dir/ *.log venv/ **/tmp +result* +.direnv/ # Deployment assets crates/walrus-orchestrator/assets/ diff --git a/crates/walrus-sdk/src/config.rs b/crates/walrus-sdk/src/config.rs index e28bf1996..433ddbe37 100644 --- a/crates/walrus-sdk/src/config.rs +++ b/crates/walrus-sdk/src/config.rs @@ -38,19 +38,23 @@ pub use self::{ /// Returns the default paths for the Walrus configuration file. pub fn default_configuration_paths() -> Vec { const WALRUS_CONFIG_FILE_NAMES: [&str; 2] = ["client_config.yaml", "client_config.yml"]; + let mut rv = Vec::new(); let mut directories = vec![PathBuf::from(".")]; - if let Ok(xdg_config_dir) = std::env::var("XDG_CONFIG_HOME") { + if let Ok(xdg_config_dir) = env::var("XDG_CONFIG_HOME") { directories.push(xdg_config_dir.into()); } if let Some(home_dir) = home::home_dir() { directories.push(home_dir.join(".config").join("walrus")); directories.push(home_dir.join(".walrus")); } - directories + rv.extend(directories .into_iter() .cartesian_product(WALRUS_CONFIG_FILE_NAMES) - .map(|(directory, file_name)| directory.join(file_name)) - .collect() + .map(|(directory, file_name)| directory.join(file_name))); + if let Ok(wal_config) = env::var("WALRUS_CONFIG") { + rv.push(PathBuf::from(wal_config)); + } + rv } /// Loads the Walrus configuration from the given path and context. diff --git a/crates/walrus-service/src/client/cli/args.rs b/crates/walrus-service/src/client/cli/args.rs index 0aeaa9ad1..ca6bcc197 100644 --- a/crates/walrus-service/src/client/cli/args.rs +++ b/crates/walrus-service/src/client/cli/args.rs @@ -50,6 +50,7 @@ pub struct App { /// 2. If the environment variable `XDG_CONFIG_HOME` is set, in `$XDG_CONFIG_HOME/walrus/`. /// 3. In `~/.config/walrus/`. /// 4. In `~/.walrus/`. + /// 5. The environment variable `WALRUS_CONFIG` as complete path // NB: Keep this in sync with `crate::cli`. #[arg(long, verbatim_doc_comment, global = true)] #[serde( diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..21d98bbc0 --- /dev/null +++ b/flake.lock @@ -0,0 +1,100 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1742452566, + "narHash": "sha256-sVuLDQ2UIWfXUBbctzrZrXM2X05YjX08K7XHMztt36E=", + "owner": "nix-community", + "repo": "fenix", + "rev": "7d9ba794daf5e8cc7ee728859bc688d8e26d5f06", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1742827005, + "narHash": "sha256-Wzzjl/WgUFiwW7Ljw2xbvikzsWMzkfkEv7zG7Bh2qEg=", + "owner": "poelzi", + "repo": "nixpkgs", + "rev": "0ada49bd3679ec0f930d6ec7d55d9ddd12d9fdb4", + "type": "github" + }, + "original": { + "owner": "poelzi", + "ref": "fetch-cargo-vendor-dup", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1742296961, + "narHash": "sha256-gCpvEQOrugHWLimD1wTFOJHagnSEP6VYBDspq96Idu0=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "15d87419f1a123d8f888d608129c3ce3ff8f13d4", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..aff096566 --- /dev/null +++ b/flake.nix @@ -0,0 +1,364 @@ +{ + inputs = { + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + #nixpkgs.url = "nixpkgs/nixos-unstable"; + # we need to wait for https://github.com/NixOS/nixpkgs/pull/387337 + # nixpkgs.url = "github:TomaSajt/nixpkgs?ref=fetch-cargo-vendor-dup"; + # rebased version on master + nixpkgs.url = "github:poelzi/nixpkgs?ref=fetch-cargo-vendor-dup"; + + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = + { + self, + fenix, + flake-utils, + nixpkgs, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + toolchain = fenix.packages.${system}.fromToolchainFile { + file = ./rust-toolchain.toml; + # update when toolchain changes + sha256 = "sha256-X/4ZBHO3iW0fOenQ3foEvscgAPJYl2abspaBThDOukI="; + }; + # update this hash when dependencies changes + cargoHash = "sha256-MBeVZ6xTwokR6lwexE6hfne4U2oB7rD1dFBVI07ftt0="; + # cargoHash = ""; + + ############## + pkgs = nixpkgs.legacyPackages.${system}; + walrusVersion = (builtins.fromTOML (builtins.readFile ./Cargo.toml)).workspace.package.version; + llvmVersion = pkgs.llvmPackages_14; + in + let + lib = pkgs.lib; + # we use the llvm toolchain + stdenv = llvmVersion.stdenv; + platform = pkgs.makeRustPlatform { + cargo = toolchain; + rustc = toolchain; + }; + nativeBuildInputs = with pkgs; [ + git + pkg-config + makeWrapper + llvmVersion.clang + platform.bindgenHook + #bintools + llvmVersion.bintools + ]; + basePkgs = with pkgs; [ + zstd + ]; + contractsPkg = pkgs.stdenvNoCC.mkDerivation { + name = "walrus-contracts"; + version = walrusVersion; + + src = ./contracts; + + installPhase = '' + mkdir -p $out + cp -r $src/* $out/ + ''; + }; + configPkg = pkgs.stdenvNoCC.mkDerivation { + name = "walrus-config"; + version = walrusVersion; + + src = ./setup; + + installPhase = '' + mkdir -p $out + cp -r $src/* $out/ + ''; + }; + + in + let + # builts a sui rust crate + mkCrate = + { + name ? null, + path ? null, + bins ? null, + contracts ? false, + wrapConfigPkg ? null, + extraDeps ? [ ], + features ? [ ], + noDefaultFeatures ? false, + profile ? "release", + mainProgram ? null, + }: + (platform.buildRustPackage { + pname = + if !builtins.isNull name then + name + else if !builtins.isNull path then + path + else + "sui"; + version = walrusVersion; + inherit nativeBuildInputs toolchain cargoHash; # platform; + + src = lib.fileset.toSource { + root = ./.; + fileset = ( + lib.fileset.unions [ + ./Cargo.toml + ./Cargo.lock + ./crates + ./contracts + ] + ); + }; + + cargoBuildFlags = + (lib.lists.optionals (!builtins.isNull path) [ + "-p" + path + ]) + ++ (lib.optionals (!builtins.isNull bins) ( + lib.lists.concatMap (x: [ + "--bin" + x + ]) bins + )); + buildNoDefaultFeatures = noDefaultFeatures; + buildFeatures = features; + + buildType = profile; + + useFetchCargoVendor = true; + + buildInputs = + basePkgs + ++ lib.optionals stdenv.isDarwin ( + with pkgs; + [ + darwin.apple_sdk.frameworks.CoreFoundation + darwin.apple_sdk.frameworks.CoreServices + darwin.apple_sdk.frameworks.IOKit + darwin.apple_sdk.frameworks.Security + darwin.apple_sdk.frameworks.SystemConfiguration + ] + ) + ++ extraDeps + ++ lib.optional contracts contractsPkg + ++ lib.optional (!isNull wrapConfigPkg) wrapConfigPkg; + + preBuild = '' + export GIT_REVISION="${self.rev or self.dirtyRev or "dirty"}"; + ''; + + postFixup = if (!isNull wrapConfigPkg) then '' + if [ -e $out/bin/walrus ]; then + wrapProgram $out/bin/walrus --set WALRUS_CONFIG ${wrapConfigPkg}/client_config.yaml; + fi + '' else ""; + + doCheck = true; + useNextest = true; + + env = { + ZSTD_SYS_USE_PKG_CONFIG = true; + }; + + outputs = [ "out" ]; + + meta = with pkgs.lib; { + description = "Sui, a next-generation smart contract platform with high throughput, low latency, and an asset-oriented programming model powered by the Move programming language"; + homepage = "https://github.com/mystenLabs/sui"; + changelog = "https://github.com/mystenLabs/sui/blob/${walrusVersion}/RELEASES.md"; + license = with licenses; [ + cc-by-40 + asl20 + ]; + maintainers = with maintainers; [ poelzi ]; + mainProgram = if ! isNull mainProgram then mainProgram else name; + }; + }); + mkDocker = + { + name, + tag ? (self.rev or self.dirtyRev), + tpkg, + extraPackages ? [ ], + cmd ? [ ], + labels ? { }, + rustLog ? null, + debug ? false, + contracts ? false, + }: + pkgs.dockerTools.buildImage { + # pkgs.dockerTools.buildLayeredImage { + + inherit name tag; + + copyToRoot = pkgs.buildEnv { + name = "image-${name}"; + paths = + with pkgs.dockerTools; + [ + usrBinEnv + binSh + caCertificates + fakeNss + ] + ++ [ + ( + if debug then + # FIXME: why does this not set buildType do debug ? + (tpkg.overrideAttrs { + buildType = "debug"; + separateDebugInfo = false; + }) + else + tpkg + ) + ] + ++ extraPackages + ++ lib.optional contracts contractsPkg + ++ lib.optionals debug [ + pkgs.bashInteractive + pkgs.coreutils + pkgs.gdb + ]; + pathsToLink = [ + "/bin" + "/lib" + "/share" + ]; + }; + + config = { + Cmd = cmd; + # Cmd = ["${pkgs.bash}/bin/bash"]; + WorkingDir = "/"; + Labels = { + "git-revision" = builtins.toString (self.rev or self.dirtyRev or "dirty"); + "build-date" = self.lastModifiedDate; + } // labels; + Env = (lib.optional debug "PS1=$(pwd) > ") + ++ (lib.optional (!isNull rustLog) "RUST_LOG=${rustLog}") + ++ [ "PATH=/bin:" ]; + }; + + }; + in + let + walruspkgs = { + walrus-full = mkCrate { }; + walrus-proxy = mkCrate { + name = "walrus-proxy"; + bins = [ "walrus-proxy" ]; + }; + walrus-service = mkCrate { + name = "walrus-service"; + wrapConfigPkg = configPkg; + bins = ["walrus" "walrus-node" "walrus-deploy"]; + }; + walrus-backup = mkCrate { + name = "walrus-backup"; + features = ["walrus-service/backup"]; + extraDeps = [ pkgs.postgresql ]; + bins = ["walrus-backup"]; + }; + walrus-stress = mkCrate { + name = "walrus-stress"; + genesis = true; + bins = ["walrus-stress"]; + }; + walrus = mkCrate { + name = "walrus"; + wrapConfigPkg = configPkg; + bins = ["walrus"]; + }; + # sui-light-client = mkCrate { name = "sui-light-client"; }; + }; + in + let + dockerImages = { + docker-walrus-service = { + name = "walrus-service"; + tpkg = walruspkgs.walrus-service; + contracts = true; + extraPackages = [ pkgs.curl pkgs.git ]; + }; + docker-walrus-backup = { + name = "walrus-backup"; + tpkg = walruspkgs.walrus-backup; + rustLog = "info,walrus_service::common::event_blob_downloader=warn"; + extraPackages = [ pkgs.curl pkgs.git ]; + contracts = true; + }; + docker-walrus = { + name = "walrus"; + tpkg = walruspkgs.walrus; + cmd = [ "walrus" ]; + }; + }; + in + { + apps.walrus = flake-utils.lib.mkApp { + drv = walruspkgs.walrus; + }; + devShells.default = pkgs.mkShell.override { inherit stdenv; } { + inherit nativeBuildInputs; + RUST_SRC_PATH = "${fenix.packages.${system}.stable.rust-src}/bin/rust-lib/src"; + RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache"; + # use sccache also for c + shellHook = '' + export CC="sccache $CC" + export CXX="sccache $CXX" + # make j an alias for just + alias j=just + # complete -F _just -o bashdefault -o default j + ''; + RUST_BACKTRACE = 1; + + CFLAGS = "-O2"; + CXXFLAGS = "-O2"; + buildInputs = + nativeBuildInputs + ++ [ toolchain ] + ++ (with pkgs; [ + (fenix.packages."${system}".stable.withComponents [ + "clippy" + "rustfmt" + ]) + # turbo + cargo-deny + cargo-nextest + just + mold + nixfmt-rfc-style + pnpm + python3 + sccache + git + deno + ]); + }; + packages = + walruspkgs + // { + walrus-contracts = contractsPkg; + default = walruspkgs.walrus; + } + // (lib.attrsets.mapAttrs (name: spec: (mkDocker spec)) dockerImages) + // + # define debug versions of docker images + (lib.attrsets.mapAttrs' ( + name: spec: (lib.nameValuePair (name + "-debug") (mkDocker (spec // { debug = true; }))) + ) dockerImages); + } + ); +} diff --git a/justfile b/justfile new file mode 100644 index 000000000..4ecf1dec9 --- /dev/null +++ b/justfile @@ -0,0 +1,48 @@ +#!/usr/bin/env just --justfile + +default: (list) + +list: + just --list + +# run tests +test *ARGS: + cargo nextest run {{ARGS}} + +# build docker tools image +build-docker-walrus: (build-docker "walrus") + +build-docker-walrus-service: (build-docker "walrus-service") + +build-docker-walrus-backup: (build-docker "walrus-backup") + +build-docker target: + nix build .#docker-{{target}} -L --out-link result-docker-{{target}} + +# build and load specified docker image +build-load-docker target: (build-docker target) + docker load < result-docker-{{target}} + +# build a cargo target or all +build *ARG: + mold -run cargo build {{ARG}} + +# build nix package +build-nix target: + nix build .#sui-{{target}} -L --out-link result-{{target}} + +# build developer nix package +build-dev-tools: (build-nix "dev-tools") + +# clean cargo cache +clean-cache: + cargo clean + +# clean sccache cache directory +[unix] +clean-sccache: + rm -r $(sccache --show-adv-stats | grep -e "Cache location[[:space:]]*Local disk: " | sed 's/.*: "\(.*\)"/\1/')/* || true + +# clean cargo and sccache caches +[confirm] +deep-clean: clean-cache clean-sccache \ No newline at end of file