rustPlatform.fetchCargoVendor: allow duplicated dependencies

pkgs/build-support/rust/fetch-cargo-vendor-util.py

@@ -13,6 +13,7 @@
 from urllib.parse import unquote
 
 import requests
+import tomli_w
 from requests.adapters import HTTPAdapter, Retry
 
 eprint = functools.partial(print, file=sys.stderr)
@@ -172,8 +173,12 @@ def get_manifest_metadata(manifest_path: Path) -> dict[str, Any]:
     return json.loads(output)
 
 
-def try_get_crate_manifest_path_from_mainfest_path(manifest_path: Path, crate_name: str) -> Path | None:
-    metadata = get_manifest_metadata(manifest_path)
+def try_get_crate_manifest_path_from_manifest_path(manifest_path: Path, crate_name: str) -> Path | None:
+    try:
+        metadata = get_manifest_metadata(manifest_path)
+    except subprocess.CalledProcessError:
+        eprint(f"Warning: cargo metadata failed for {manifest_path}, skipping")
+        return None
 
     for pkg in metadata["packages"]:
         if pkg["name"] == crate_name:
@@ -183,11 +188,15 @@ def try_get_crate_manifest_path_from_mainfest_path(manifest_path: Path, crate_na
 
 
 def find_crate_manifest_in_tree(tree: Path, crate_name: str) -> Path:
-    # in some cases Cargo.toml is not located at the top level, so we also look at subdirectories
-    manifest_paths = tree.glob("**/Cargo.toml")
+    # Scan all Cargo.toml files; sort by depth/path to make ordering deterministic
+    # and prefer less-nested manifests first.
+    manifest_paths = sorted(
+        tree.glob("**/Cargo.toml"),
+        key=lambda path: (len(path.parts), str(path)),
+    )
 
     for manifest_path in manifest_paths:
-        res = try_get_crate_manifest_path_from_mainfest_path(manifest_path, crate_name)
+        res = try_get_crate_manifest_path_from_manifest_path(manifest_path, crate_name)
         if res is not None:
             return res
 
@@ -256,6 +265,21 @@ def extract_crate_tarball_contents(tarball_path: Path, crate_out_dir: Path) -> N
     subprocess.check_output(cmd)
 
 
+def make_git_source_selector(source_info: GitSourceInfo) -> dict[str, str]:
+    selector = {}
+    selector["git"] = source_info["url"]
+    if source_info["type"] is not None:
+        selector[source_info["type"]] = source_info["value"]
+    return selector
+
+
+def make_registry_source_selector(source: str) -> dict[str, str]:
+    registry = source[9:] if source.startswith("registry+") else source
+    selector = {}
+    selector["registry"] = registry
+    return selector
+
+
 def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
     lockfile_path = vendor_staging_dir / "Cargo.lock"
     out_dir.mkdir(exist_ok=True)
@@ -264,28 +288,83 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
     cargo_lock_toml = load_toml(lockfile_path)
     lockfile_version = get_lockfile_version(cargo_lock_toml)
 
-    config_lines = [
-        '[source.vendored-sources]',
-        'directory = "@vendor@"',
-        '[source.crates-io]',
-        'replace-with = "vendored-sources"',
-    ]
+    source_to_ind: dict[str, str] = {}
+    source_config = {}
+    next_registry_ind = 0
+    next_git_ind = 0
+
+    def add_source_replacement(
+        orig_key: str,
+        orig_selector: dict[str, str],
+        vendored_key: str,
+        vendored_dir: str
+    ) -> None:
+        source_config[vendored_key] = {}
+        source_config[vendored_key]["directory"] = vendored_dir
+        source_config[orig_key] = orig_selector
+        source_config[orig_key]["replace-with"] = vendored_key
+
+    # we reserve registry index 0 for crates-io
+    source_to_ind["registry+https://github.com/rust-lang/crates.io-index"] = "registry-0"
+    source_to_ind["sparse+https://index.crates.io/"] = "registry-0"
+    add_source_replacement(
+        orig_key="crates-io",
+        orig_selector={},  # there is an internal selector defined for the `crates-io` source
+        vendored_key="vendored-source-registry-0",
+        vendored_dir="@vendor@/source-registry-0"
+    )
+    next_registry_ind += 1
+
+    for pkg in cargo_lock_toml["package"]:
+        # ignore local dependencies
+        if "source" not in pkg.keys():
+            continue
+        source: str = pkg["source"]
+        if source in source_to_ind:
+            continue
+
+        if source.startswith("git+"):
+            ind = f"git-{next_git_ind}"
+            next_git_ind += 1
+            source_info = parse_git_source(source, lockfile_version)
+            selector = make_git_source_selector(source_info)
+        elif source.startswith("registry+") or source.startswith("sparse+"):
+            ind = f"registry-{next_registry_ind}"
+            next_registry_ind += 1
+            selector = make_registry_source_selector(source)
+        else:
+            raise Exception(f"Can't process source: {source}.")
+
+        source_to_ind[source] = ind
+        add_source_replacement(
+            orig_key=f"original-source-{ind}",
+            orig_selector=selector,
+            vendored_key=f"vendored-source-{ind}",
+            vendored_dir=f"@vendor@/source-{ind}"
+        )
+
+    config_path = out_dir / ".cargo" / "config.toml"
+    config_path.parent.mkdir()
+
+    with open(config_path, "wb") as config_file:
+        tomli_w.dump({"source": source_config}, config_file)
 
-    seen_source_keys = set()
     for pkg in cargo_lock_toml["package"]:
 
         # ignore local dependenices
         if "source" not in pkg.keys():
             continue
 
         source: str = pkg["source"]
-
-        dir_name = f"{pkg["name"]}-{pkg["version"]}"
-        crate_out_dir = out_dir / dir_name
+        source_ind = source_to_ind[source]
+        crate_dir_name = f"{pkg["name"]}-{pkg["version"]}"
+        source_dir_name = f"source-{source_ind}"
+        crate_out_dir = out_dir / source_dir_name / crate_dir_name
+        crate_out_dir.parent.mkdir(exist_ok=True)
 
         if source.startswith("git+"):
 
-            source_info = parse_git_source(pkg["source"], lockfile_version)
+            source_info = parse_git_source(source, lockfile_version)
 
             git_sha_rev = source_info["git_sha_rev"]
             git_tree = vendor_staging_dir / "git" / git_sha_rev
@@ -296,23 +375,13 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
             with open(crate_out_dir / ".cargo-checksum.json", "w") as f:
                 json.dump({"files": {}}, f)
 
-            source_key = source[0:source.find("#")]
-
-            if source_key in seen_source_keys:
-                continue
-
-            seen_source_keys.add(source_key)
+        elif source.startswith("registry+") or source.startswith("sparse+"):
+            filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"
 
-            config_lines.append(f'[source."{source_key}"]')
-            config_lines.append(f'git = "{source_info["url"]}"')
-            if source_info["type"] is not None:
-                config_lines.append(f'{source_info["type"]} = "{source_info["value"]}"')
-            config_lines.append('replace-with = "vendored-sources"')
+            # TODO: change this when non-crates-io registries are supported
+            dir_name = "tarballs"
 
-        elif source.startswith("registry+"):
-
-            filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"
-            tarball_path = vendor_staging_dir / "tarballs" / filename
+            tarball_path = vendor_staging_dir / dir_name / filename
 
             extract_crate_tarball_contents(tarball_path, crate_out_dir)
 
@@ -323,10 +392,6 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
         else:
             raise Exception(f"Can't process source: {source}.")
 
-    (out_dir / ".cargo").mkdir()
-    with open(out_dir / ".cargo" / "config.toml", "w") as config_file:
-        config_file.writelines(line + "\n" for line in config_lines)
-
 
 def main() -> None:
     subcommand = sys.argv[1]

pkgs/build-support/rust/fetch-cargo-vendor.nix

@@ -26,6 +26,7 @@ let
       with python3Packages;
       [
         requests
+        tomli-w
       ]
       ++ requests.optional-dependencies.socks; # to support socks proxy envs like ALL_PROXY in requests
     flakeIgnore = [

pkgs/by-name/ca/cargo-tauri/test-app.nix

@@ -19,7 +19,7 @@ stdenv.mkDerivation (finalAttrs: {
   inherit (cargo-tauri) version src;
 
   postPatch = lib.optionalString stdenv.hostPlatform.isLinux ''
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
   '';
 

pkgs/by-name/ce/celeste/package.nix

@@ -35,7 +35,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
   patches = [ ./missing-unsafe-block.patch ];
 
   postPatch = ''
-    pushd $cargoDepsCopy/librclone-sys-*
+    pushd $cargoDepsCopy/*/librclone-sys-*
     oldHash=$(sha256sum build.rs | cut -d " " -f 1)
     patch -p2 < ${./librclone-path.patch}
     substituteInPlace build.rs \

pkgs/by-name/cl/clash-verge-rev/unwrapped.nix

@@ -64,17 +64,17 @@ rustPlatform.buildRustPackage {
     sed -i -e '/Mihomo Alpha/d' ./src/components/setting/mods/clash-core-viewer.tsx
 
     # Set service.sock path
-    substituteInPlace $cargoDepsCopy/clash_verge_service_ipc-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/clash_verge_service_ipc-*/src/lib.rs \
       --replace-fail "/tmp/verge/clash-verge-service.sock" "/run/clash-verge-rev/service.sock"
     # Set verge-mihomo.sock path
     substituteInPlace src-tauri/src/utils/dirs.rs \
       --replace-fail 'once("/tmp")' 'once(&std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| std::env::var("UID").map(|uid| format!("/run/user/{}", uid)).unwrap_or_else(|_| "/tmp".to_string())))' \
       --replace-fail 'join("verge")' 'join("clash-verge-rev")'
 
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
 
-    substituteInPlace $cargoDepsCopy/sysproxy-*/src/linux.rs \
+    substituteInPlace $cargoDepsCopy/*/sysproxy-*/src/linux.rs \
       --replace-fail '"gsettings"' '"${glib.bin}/bin/gsettings"' \
       --replace-fail '"kreadconfig5"' '"${libsForQt5.kconfig}/bin/kreadconfig5"' \
       --replace-fail '"kreadconfig6"' '"${kdePackages.kconfig}/bin/kreadconfig6"' \

pkgs/by-name/do/dorion/package.nix

@@ -120,7 +120,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
     # remove updater
     rm -rf updater
 
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
 
     # disable pre-build script and disable auto-updater

pkgs/by-name/fe/ferron/package.nix

@@ -21,7 +21,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
 
   # ../../ is cargoDepsCopy, and obviously does not contain monoio's README.md
   postPatch = ''
-    substituteInPlace $cargoDepsCopy/monoio-0.2.4/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/monoio-0.2.4/src/lib.rs \
       --replace-fail '#![doc = include_str!("../../README.md")]' ""
   '';
 

pkgs/by-name/fi/firecracker/package.nix

@@ -36,7 +36,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
   #
   # For seccompiler: fix hardcoded /usr/local/lib path to libseccomp.lib, this makes sure rustc can find seccomp across stdenv's(including pkgsStatic).
   postPatch = ''
-    substituteInPlace $cargoDepsCopy/aws-lc-sys-*/aws-lc/crypto/asn1/a_bitstr.c \
+    substituteInPlace $cargoDepsCopy/*/aws-lc-sys-*/aws-lc/crypto/asn1/a_bitstr.c \
       --replace-warn '(len > INT_MAX - 1)' '(len < 0 || len > INT_MAX - 1)'
 
     substituteInPlace src/cpu-template-helper/build.rs \

pkgs/by-name/li/lightway/package.nix

@@ -26,7 +26,7 @@ rustPlatform.buildRustPackage {
   # Drop when Lightway bumps wolfSSL past commit 5c2c459, or > 5.8.4.
   postPatch = ''
     patch -Np1 \
-      -d $cargoDepsCopy/wolfssl-sys-2.0.0/wolfssl-src \
+      -d $cargoDepsCopy/*/wolfssl-sys-2.0.0/wolfssl-src \
       -i ${./backport-darwin-address-calc-fix.patch}
   '';
 

pkgs/by-name/no/noriskclient-launcher-unwrapped/package.nix

@@ -41,7 +41,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
   ];
 
   postPatch = ''
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
   '';
 

pkgs/by-name/os/oscavmgr/package.nix

@@ -36,7 +36,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
   doInstallCheck = true;
 
   postPatch = ''
-    alvr_session=$(echo $cargoDepsCopy/alvr_session-*/)
+    alvr_session=$(echo $cargoDepsCopy/*/alvr_session-*/)
     substituteInPlace "$alvr_session/build.rs" \
       --replace-fail \
         'alvr_filesystem::workspace_dir().join("openvr/headers/openvr_driver.h")' \

pkgs/by-name/ov/overlayed/package.nix

@@ -62,7 +62,7 @@ rustPlatform.buildRustPackage rec {
   };
 
   postPatch = ''
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
 
     # disable updater

pkgs/by-name/pa/pagefind/package.nix

@@ -127,24 +127,24 @@ rustPlatform.buildRustPackage (finalAttrs: {
     # patch build-time dependency downloads
     (
       # add support for file:// urls
-      patch -d $cargoDepsCopy/lindera-dictionary-*/ -p1 < ${./lindera-dictionary-support-file-paths.patch}
+      patch -d $cargoDepsCopy/*/lindera-dictionary-*/ -p1 < ${./lindera-dictionary-support-file-paths.patch}
 
       # patch urls
       ${lib.pipe finalAttrs.passthru.lindera-srcs [
         (lib.mapAttrsToList (
           key: src: ''
             # compgen is only in bashInteractive
-            declare -a expanded_glob=($cargoDepsCopy/${src.vendorDir}/build.rs)
+            declare -a expanded_glob=($cargoDepsCopy/*/${src.vendorDir}/build.rs)
             if [[ "''${#expanded_glob[@]}" -eq 0 ]]; then
-              echo >&2 "ERROR: '$cargoDepsCopy/${src.vendorDir}/build.rs' not found! (pagefind.passthru.lindera-srcs.${key})"
+              echo >&2 "ERROR: '$cargoDepsCopy/*/${src.vendorDir}/build.rs' not found! (pagefind.passthru.lindera-srcs.${key})"
               false
             elif [[ "''${#expanded_glob[@]}" -gt 1 ]]; then
-              echo >&2 "ERROR: '$cargoDepsCopy/${src.vendorDir}/build.rs' matches more than one file! (pagefind.passthru.lindera-srcs.${key})"
+              echo >&2 "ERROR: '$cargoDepsCopy/*/${src.vendorDir}/build.rs' matches more than one file! (pagefind.passthru.lindera-srcs.${key})"
               printf >&2 "match: %s\n" "''${expanded_glob[@]}"
               false
             fi
-            echo "patching $cargoDepsCopy/${src.vendorDir}/build.rs..."
-            substituteInPlace $cargoDepsCopy/${src.vendorDir}/build.rs --replace-fail "${src.url}" "file://${src}"
+            echo "patching $cargoDepsCopy/*/${src.vendorDir}/build.rs..."
+            substituteInPlace $cargoDepsCopy/*/${src.vendorDir}/build.rs --replace-fail "${src.url}" "file://${src}"
             unset expanded_glob
           ''
         ))

pkgs/by-name/qu/quantframe/package.nix

@@ -29,7 +29,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
   };
 
   postPatch = ''
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
 
     substituteInPlace src-tauri/tauri.conf.json \

pkgs/by-name/rq/rquickshare/package.nix

@@ -36,7 +36,7 @@ rustPlatform.buildRustPackage rec {
 
   # from https://github.com/NixOS/nixpkgs/blob/04e40bca2a68d7ca85f1c47f00598abb062a8b12/pkgs/by-name/ca/cargo-tauri/test-app.nix#L23-L26
   postPatch = lib.optionalString stdenv.hostPlatform.isLinux ''
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
   '';
 

pkgs/by-name/ru/rustdesk-flutter/package.nix

@@ -162,12 +162,12 @@ flutter329.buildFlutterApplication rec {
   postPatch = ''
     cd flutter
     if [ $cargoDepsCopy ]; then # That will be inherited to buildDartPackage and it doesn't have cargoDepsCopy
-      substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+      substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
         --replace-fail "libayatana-appindicator3.so.1" "${lib.getLib libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
       # Disable static linking of ffmpeg since https://github.com/21pages/hwcodec/commit/1873c34e3da070a462540f61c0b782b7ab15dc84
-      sed -i 's/static=//g' $cargoDepsCopy/hwcodec-*/build.rs
-      sed -e '1i #include <cstdint>' -i $cargoDepsCopy/webm-1.1.0/src/sys/libwebm/mkvparser/mkvparser.cc
-      sed -e '1i #include <cstdint>' -i $cargoDepsCopy/webm-sys-1.0.4/libwebm/mkvparser/mkvparser.cc
+      sed -i 's/static=//g' $cargoDepsCopy/*/hwcodec-*/build.rs
+      sed -e '1i #include <cstdint>' -i $cargoDepsCopy/*/webm-1.1.0/src/sys/libwebm/mkvparser/mkvparser.cc
+      sed -e '1i #include <cstdint>' -i $cargoDepsCopy/*/webm-sys-1.0.4/libwebm/mkvparser/mkvparser.cc
     fi
 
     substituteInPlace ../Cargo.toml --replace-fail ", \"staticlib\", \"rlib\"" ""

pkgs/by-name/ru/rustdesk/package.nix

@@ -112,8 +112,8 @@ rustPlatform.buildRustPackage (finalAttrs: {
   ];
 
   postPatch = ''
-    sed -e '1i #include <cstdint>' -i $cargoDepsCopy/webm-1.1.0/src/sys/libwebm/mkvparser/mkvparser.cc
-    sed -e '1i #include <cstdint>' -i $cargoDepsCopy/webm-sys-1.0.4/libwebm/mkvparser/mkvparser.cc
+    sed -e '1i #include <cstdint>' -i $cargoDepsCopy/*/webm-1.1.0/src/sys/libwebm/mkvparser/mkvparser.cc
+    sed -e '1i #include <cstdint>' -i $cargoDepsCopy/*/webm-sys-1.0.4/libwebm/mkvparser/mkvparser.cc
   '';
 
   # Add static ui resources and libsciter to same folder as binary so that it

pkgs/by-name/sl/slimevr/package.nix

@@ -85,7 +85,7 @@ rustPlatform.buildRustPackage rec {
   ''
   + lib.optionalString stdenv.hostPlatform.isLinux ''
     # Both libappindicator-rs and SlimeVR need to know where Nix's appindicator lib is.
-    substituteInPlace $cargoDepsCopy/libappindicator-sys-*/src/lib.rs \
+    substituteInPlace $cargoDepsCopy/*/libappindicator-sys-*/src/lib.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"
     substituteInPlace gui/src-tauri/src/tray.rs \
       --replace-fail "libayatana-appindicator3.so.1" "${libayatana-appindicator}/lib/libayatana-appindicator3.so.1"