ci: check MSRV correctly (apache#849)

Author: xxchan

.github/actions/setup-builder/action.yml

@@ -21,18 +21,24 @@ name: Prepare Rust Builder
 description: 'Prepare Rust Build Environment'
 inputs:
   rust-version:
-    description: 'version of rust to install (e.g. stable)'
-    required: true
-    default: 'stable'
+    description: 'version of rust to install and use'
 runs:
   using: "composite"
   steps:
-    - name: Setup Rust toolchain
+    - name: Setup specified Rust toolchain
       shell: bash
+      if: ${{ inputs.rust-version != '' }}
       run: |
         echo "Installing ${{ inputs.rust-version }}"
         rustup toolchain install ${{ inputs.rust-version }}
-        rustup default ${{ inputs.rust-version }}
+        rustup override set ${{ inputs.rust-version }}
+        rustup component add rustfmt clippy
+    - name: Setup Rust toolchain according to rust-toolchain.toml
+      shell: bash
+      if: ${{ inputs.rust-version == '' }}
+      run: |
+        echo "Installing toolchain according to rust-toolchain.toml"
+        rustup show
         rustup component add rustfmt clippy
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766

.github/workflows/ci.yml

@@ -43,6 +43,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
       - name: Check License Header
         uses: apache/skywalking-eyes/[email protected]
 
@@ -89,8 +92,6 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{ env.rust_msrv }}
 
       - name: Cache Rust artifacts
         uses: Swatinem/rust-cache@v2
@@ -109,6 +110,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+
       - name: Cache Rust artifacts
         uses: Swatinem/rust-cache@v2
 
@@ -122,8 +126,6 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: ${{ env.rust_msrv }}
 
       - name: Cache Rust artifacts
         uses: Swatinem/rust-cache@v2
@@ -136,3 +138,24 @@ jobs:
 
       - name: Doc Test
         run: cargo test --no-fail-fast --doc --all-features --workspace
+
+  msrv:
+    name: Verify MSRV
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Nightly Rust toolchain
+        uses: ./.github/actions/setup-builder
+      - name: Generate minimal versions lockfile
+        run: |
+          cargo generate-lockfile -Z direct-minimal-versions -Z minimal-versions
+          # Some dependencies don't correctly specify a minimal version for their dependencies and will fail to build.
+          # So we update these transitive dependencies here.
+          cargo update tap faststr metainfo linkedbytes
+      - name: Setup MSRV Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: ${{ env.rust_msrv }}
+      - name: Check MSRV
+        run: |
+          cargo +${{ env.rust_msrv }} check --locked --workspace --exclude iceberg-datafusion --exclude iceberg-catalog-s3tables

Cargo.toml

@@ -41,62 +41,62 @@ rust-version = "1.77.1"
 anyhow = "1.0.72"
 apache-avro = "0.17"
 array-init = "2"
-arrow-arith = { version = "53" }
-arrow-array = { version = "53" }
-arrow-cast = { version = "53" }
-arrow-ord = { version = "53" }
-arrow-schema = { version = "53" }
-arrow-select = { version = "53" }
-arrow-string = { version = "53" }
+arrow-arith = { version = "53.3.0" }
+arrow-array = { version = "53.3.0" }
+arrow-cast = { version = "53.3.0" }
+arrow-ord = { version = "53.3.0" }
+arrow-schema = { version = "53.3.0" }
+arrow-select = { version = "53.3.0" }
+arrow-string = { version = "53.3.0" }
 async-stream = "0.3.5"
-async-trait = "0.1"
+async-trait = "0.1.73"
 async-std = "1.12"
-aws-config = "1.5.13"
-aws-sdk-glue = "1.76"
+aws-config = "1"
+aws-sdk-glue = "1.39"
 bimap = "0.6"
 bitvec = "1.0.1"
-bytes = "1.5"
-chrono = "0.4.34"
+bytes = "1.6"
+chrono = "0.4.38"
 ctor = "0.2.8"
 derive_builder = "0.20"
 either = "1"
 env_logger = "0.11.0"
-fnv = "1"
+fnv = "1.0.7"
 futures = "0.3"
 iceberg = { version = "0.4.0", path = "./crates/iceberg" }
 iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" }
 iceberg-catalog-hms = { version = "0.4.0", path = "./crates/catalog/hms" }
 iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" }
 iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" }
 itertools = "0.13"
-log = "0.4"
+log = "0.4.22"
 mockito = "1"
 murmur3 = "0.5.2"
 num-bigint = "0.4.6"
-once_cell = "1"
+once_cell = "1.19"
 opendal = "0.51.0"
 ordered-float = "4"
-parquet = "53.1"
-paste = "1"
+parquet = "53.3.0"
+paste = "1.0.15"
 pilota = "0.11.2"
 pretty_assertions = "1.4"
 port_scanner = "0.1.5"
-rand = "0.8"
+rand = "0.8.5"
 regex = "1.10.5"
-reqwest = { version = "0.12", default-features = false, features = ["json"] }
+reqwest = { version = "0.12.2", default-features = false, features = ["json"] }
 rust_decimal = "1.31"
-serde = { version = "1", features = ["rc"] }
-serde_bytes = "0.11.8"
-serde_derive = "1"
-serde_json = "1"
+serde = { version = "1.0.204", features = ["rc"] }
+serde_bytes = "0.11.15"
+serde_derive = "1.0.204"
+serde_json = "1.0.120"
 serde_repr = "0.1.16"
 serde_with = "3.4"
 tempfile = "3.15"
-tokio = { version = "1", default-features = false }
+tokio = { version = "1.36", default-features = false }
 typed-builder = "0.20"
-url = "2"
+url = "2.2.2"
 urlencoding = "2"
-uuid = { version = "1.6.1", features = ["v7"] }
+uuid = { version = "1.10.0", features = ["v7"] }
 volo-thrift = "0.10"
 hive_metastore = "0.1"
 tera = "1"

crates/catalog/rest/Cargo.toml

@@ -35,7 +35,7 @@ chrono = { workspace = true }
 http = "1.1.0"
 iceberg = { workspace = true }
 itertools = { workspace = true }
-log = "0.4.20"
+log = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }

crates/catalog/s3tables/Cargo.toml

@@ -20,7 +20,7 @@ name = "iceberg-catalog-s3tables"
 version = { workspace = true }
 edition = { workspace = true }
 homepage = { workspace = true }
-rust-version = { workspace = true }
+rust-version = "1.81.0"
 
 categories = ["database"]
 description = "Apache Iceberg Rust S3Tables Catalog"

crates/catalog/sql/Cargo.toml

@@ -40,7 +40,7 @@ uuid = { workspace = true, features = ["v4"] }
 iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
 itertools = { workspace = true }
 regex = "1.10.5"
-sqlx = { version = "0.8.0", features = [
+sqlx = { version = "0.8.1", features = [
   "tls-rustls",
   "runtime-tokio",
   "any",

crates/examples/Cargo.toml

@@ -27,7 +27,7 @@ rust-version = { workspace = true }
 [dependencies]
 iceberg = { workspace = true }
 iceberg-catalog-rest = { workspace = true }
-tokio = { version = "1", features = ["full"] }
+tokio = { workspace = true, features = ["full"] }
 
 [[example]]
 name = "rest-catalog-namespace"

crates/iceberg/src/io/object_cache.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::mem::size_of_val;
 use std::sync::Arc;
 
 use crate::io::FileIO;

crates/iceberg/src/utils.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::num::NonZero;
+use std::num::NonZeroUsize;
 
 // Use a default value of 1 as the safest option.
 // See https://doc.rust-lang.org/std/thread/fn.available_parallelism.html#limitations
@@ -31,12 +31,12 @@ const DEFAULT_PARALLELISM: usize = 1;
 /// are circumstances where the level of available
 /// parallelism can change during the lifetime of an executing
 /// process, but this should not be called in a hot loop.
-pub(crate) fn available_parallelism() -> NonZero<usize> {
+pub(crate) fn available_parallelism() -> NonZeroUsize {
     std::thread::available_parallelism().unwrap_or_else(|_err| {
         // Failed to get the level of parallelism.
         // TODO: log/trace when this fallback occurs.
 
         // Using a default value.
-        NonZero::new(DEFAULT_PARALLELISM).unwrap()
+        NonZeroUsize::new(DEFAULT_PARALLELISM).unwrap()
     })
 }

crates/integrations/datafusion/Cargo.toml

@@ -20,7 +20,10 @@ name = "iceberg-datafusion"
 version = { workspace = true }
 edition = { workspace = true }
 homepage = { workspace = true }
-rust-version = { workspace = true }
+# kept the same as DataFusion's MSRV
+# https://github.com/apache/datafusion?tab=readme-ov-file#rust-version-compatibility-policy
+# https://github.com/apache/datafusion/blob/main/Cargo.toml#L68
+rust-version = "1.80.1"
 
 categories = ["database"]
 description = "Apache Iceberg DataFusion Integration"