Skip to content

Commit 52d4361

Browse files
authored
Merge pull request #137 from mulimoen/feature/mpi
MPI support
2 parents b4c5100 + b5b8277 commit 52d4361

File tree

16 files changed

+315
-0
lines changed

16 files changed

+315
-0
lines changed

.github/workflows/ci.yml

+30
Original file line numberDiff line numberDiff line change
@@ -148,3 +148,33 @@ jobs:
148148
RUSTFLAGS: "-Z sanitizer=address"
149149
RUSTDOCFLAGS: "-Z sanitizer=address"
150150
run: cargo test --features netcdf-sys/static,netcdf/derive --target x86_64-unknown-linux-gnu --workspace --exclude netcdf-derive
151+
152+
mpi:
153+
name: mpi-runner
154+
runs-on: ubuntu-latest
155+
env:
156+
NETCDF_DIR: /usr/lib/x86_64-linux-gnu/netcdf/mpi/
157+
steps:
158+
- name: Checkout repository
159+
uses: actions/checkout@v4
160+
with: {submodules: false}
161+
162+
- name: Install netcdf
163+
run: sudo apt-get update && sudo apt-get install libnetcdf-mpi-dev libhdf5-openmpi-dev
164+
165+
- name: Install rust
166+
uses: dtolnay/rust-toolchain@stable
167+
with:
168+
toolchain: "nightly"
169+
170+
- name: Build
171+
run: cargo build --verbose --workspace --exclude netcdf-src --features netcdf/mpi,derive
172+
173+
- name: Test
174+
run: cargo test --verbose --workspace --exclude netcdf-src --features netcdf/mpi,derive
175+
176+
- name: Run example
177+
run: cargo run --verbose --package netcdf-examples --features mpi
178+
179+
- name: Run example in parallel
180+
run: mpirun -np 10 --oversubscribe -- target/debug/netcdf-examples

Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ members = [
44
"netcdf-sys",
55
"netcdf-src",
66
"netcdf-derive",
7+
"netcdf-examples",
78
]
89
default-members = ["netcdf", "netcdf-sys"]
910
resolver = "2"
@@ -14,3 +15,4 @@ netcdf-sys = { path = "netcdf-sys", version = "0.6.2" }
1415
netcdf-src = { path = "netcdf-src", version = "0.3.6" }
1516
netcdf-derive = { path = "netcdf-derive", version = "0.1.0" }
1617
hdf5-sys = { version = "0.8.0" }
18+
mpi-sys = { version = "0.2.1" }

netcdf-examples/Cargo.toml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[package]
2+
name = "netcdf-examples"
3+
version = "0.1.0"
4+
edition = "2021"
5+
publish = false
6+
7+
[features]
8+
mpi = ["dep:mpi", "netcdf/mpi", "dep:mpi-sys"]
9+
10+
[dependencies]
11+
netcdf = { workspace = true }
12+
mpi = { version = "0.7.0", optional = true }
13+
mpi-sys = { workspace = true, optional = true }
14+
ndarray = "0.15.6"

netcdf-examples/src/main.rs

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#[cfg(feature = "mpi")]
2+
mod parallel;
3+
4+
fn main() {
5+
#[cfg(feature = "mpi")]
6+
parallel::main().unwrap();
7+
8+
#[cfg(not(feature = "mpi"))]
9+
println!("MPI support is not included, will not run this example");
10+
}

netcdf-examples/src/parallel.rs

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
use mpi::traits::{AsRaw, Communicator};
2+
3+
fn target_function(rank: i32, t: usize) -> i32 {
4+
100 * (t as i32) + rank
5+
}
6+
7+
fn mpi_null_info() -> mpi_sys::MPI_Info {
8+
let mut info = std::ptr::null_mut();
9+
let e = unsafe { mpi_sys::MPI_Info_create(&mut info) };
10+
assert_eq!(e, mpi_sys::MPI_SUCCESS.try_into().unwrap());
11+
12+
info
13+
}
14+
15+
fn create(
16+
path: &str,
17+
communicator: impl Communicator + AsRaw<Raw = mpi_sys::MPI_Comm>,
18+
) -> Result<(), Box<dyn std::error::Error>> {
19+
let info = mpi_null_info();
20+
let mut file =
21+
netcdf::create_par_with(path, communicator.as_raw(), info, netcdf::Options::NETCDF4)?;
22+
23+
let size = communicator.size() as usize;
24+
let rank = communicator.rank();
25+
26+
file.add_dimension("x", size)?;
27+
file.add_unlimited_dimension("t")?;
28+
let var = file.add_variable::<i32>("output", &["t", "x"])?;
29+
var.access_collective()?;
30+
31+
file.enddef()?;
32+
33+
let mut var = file.variable_mut("output").unwrap();
34+
35+
let values = ndarray::Array1::from_shape_fn(10, |t| target_function(rank, t));
36+
var.put((.., rank as usize), values.view())?;
37+
38+
Ok(())
39+
}
40+
41+
fn read(
42+
path: &str,
43+
communicator: impl Communicator + AsRaw<Raw = mpi_sys::MPI_Comm>,
44+
) -> Result<(), Box<dyn std::error::Error>> {
45+
let info = mpi_null_info();
46+
47+
let file = netcdf::open_par_with(path, communicator.as_raw(), info, netcdf::Options::empty())?;
48+
49+
let rank = communicator.rank();
50+
let var = file.variable("output").unwrap();
51+
var.access_collective()?;
52+
let values = var.get::<i32, _>((.., rank as usize))?;
53+
54+
for (t, &v) in values.iter().enumerate() {
55+
assert_eq!(v, target_function(rank, t));
56+
}
57+
Ok(())
58+
}
59+
60+
pub fn main() -> Result<(), Box<dyn std::error::Error>> {
61+
let universe = mpi::initialize().unwrap();
62+
let path = "par.nc";
63+
64+
create(path, universe.world())?;
65+
66+
read(path, universe.world())?;
67+
68+
Ok(())
69+
}

netcdf-src/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ exclude = [
2929

3030
[features]
3131
dap = ["dep:link-cplusplus"]
32+
mpi = []
3233

3334
[dependencies]
3435
hdf5-sys = { workspace = true, features = ["hl", "deprecated", "zlib"] }

netcdf-src/build.rs

+4
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ fn main() {
8383
netcdf_config.define("ENABLE_BYTERANGE", "ON");
8484
}
8585

86+
if feature!("MPI").is_ok() {
87+
panic!("MPI feature was requested but the static build of netcdf does not support this");
88+
}
89+
8690
let netcdf = netcdf_config.build();
8791

8892
println!("cargo:lib=netcdf");

netcdf-sys/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ libz-sys = { version = "1.0.25" }
2424
curl-sys = { version = "0.4.51", optional = true }
2525
hdf5-sys = { workspace = true }
2626
netcdf-src = { workspace = true, optional = true }
27+
mpi-sys = { workspace = true, optional = true }
2728

2829
[dev-dependencies]
2930

@@ -32,6 +33,7 @@ default = []
3233
memio = []
3334
static = ["libz-sys/static", "hdf5-sys/static", "hdf5-sys/hl", "hdf5-sys/deprecated", "hdf5-sys/zlib", "dep:netcdf-src", "curl-sys?/static-curl", "curl-sys?/static-ssl"]
3435
dap = ["dep:curl-sys", "netcdf-src?/dap"]
36+
mpi = ["dep:mpi-sys", "netcdf-src?/mpi"]
3537

3638
[build-dependencies]
3739
semver = "1.0.9"

netcdf-sys/build.rs

+8
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,14 @@ impl NcMetaHeader {
146146
"MEMIO requested but not found in this installation of netCDF"
147147
);
148148
}
149+
if self.has_parallel {
150+
println!("cargo:rustc-cfg=feature=\"has-par\"");
151+
} else {
152+
assert!(
153+
feature!("MPI").is_err(),
154+
"MPI requested but not found in this installation of netCDF"
155+
);
156+
}
149157
}
150158
}
151159

netcdf-sys/src/lib.rs

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ mod filter;
3030
#[cfg(feature = "4.8.0")]
3131
pub use filter::*;
3232

33+
#[cfg(feature = "mpi")]
34+
pub mod par;
35+
3336
use std::sync::Mutex;
3437

3538
/// Global netCDF lock for using all functions in the netCDF library

netcdf-sys/src/par.rs

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#![cfg(feature = "mpi")]
2+
use std::ffi::{c_char, c_int};
3+
4+
use mpi_sys::{MPI_Comm, MPI_Info};
5+
6+
pub const NC_INDEPENDENT: c_int = 0;
7+
pub const NC_COLLECTIVE: c_int = 1;
8+
9+
extern "C" {
10+
pub fn nc_create_par(
11+
path: *const c_char,
12+
cmode: c_int,
13+
comm: MPI_Comm,
14+
info: MPI_Info,
15+
ncidp: *mut c_int,
16+
) -> c_int;
17+
pub fn nc_open_par(
18+
path: *const c_char,
19+
mode: c_int,
20+
comm: MPI_Comm,
21+
info: MPI_Info,
22+
ncidp: *mut c_int,
23+
) -> c_int;
24+
pub fn nc_var_par_access(ncid: c_int, varid: c_int, par_access: c_int) -> c_int;
25+
}

netcdf/Cargo.toml

+3
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,16 @@ build = "build.rs"
1919
default = ["ndarray"]
2020
static = ["netcdf-sys/static"]
2121
derive = ["dep:netcdf-derive"]
22+
mpi = ["dep:mpi-sys", "netcdf-sys/mpi"]
23+
ndarray = ["dep:ndarray"]
2224

2325
[dependencies]
2426
ndarray = { version = "0.15", optional = true }
2527
netcdf-sys = { workspace = true }
2628
netcdf-derive = { workspace = true, optional = true }
2729
bitflags = "2.4.2"
2830
libc = "0.2.155"
31+
mpi-sys = { workspace = true, optional = true }
2932

3033
[dev-dependencies]
3134
clap = { version = "4.5.1", features = ["derive"] }

netcdf/src/file.rs

+67
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,30 @@ impl RawFile {
8686
Ok(File(Self { ncid }))
8787
}
8888

89+
/// Open a `netCDF` file in read only mode in parallel mode.
90+
#[cfg(feature = "mpi")]
91+
pub(crate) fn open_par_with(
92+
path: &path::Path,
93+
communicator: mpi_sys::MPI_Comm,
94+
info: mpi_sys::MPI_Info,
95+
options: Options,
96+
) -> error::Result<File> {
97+
let f = get_ffi_from_path(path);
98+
let mut ncid: nc_type = 0;
99+
unsafe {
100+
error::checked(with_lock(|| {
101+
netcdf_sys::par::nc_open_par(
102+
f.as_ptr().cast(),
103+
options.bits(),
104+
communicator,
105+
info,
106+
&mut ncid,
107+
)
108+
}))?;
109+
}
110+
Ok(File(Self { ncid }))
111+
}
112+
89113
/// Open a `netCDF` file in append mode (read/write).
90114
pub(crate) fn append_with(path: &path::Path, options: Options) -> error::Result<FileMut> {
91115
let file = Self::open_with(path, options | Options::WRITE)?;
@@ -105,6 +129,31 @@ impl RawFile {
105129
Ok(FileMut(File(Self { ncid })))
106130
}
107131

132+
/// Create a new `netCDF` file in parallel mode
133+
#[cfg(feature = "mpi")]
134+
pub(crate) fn create_par_with(
135+
path: &path::Path,
136+
communicator: mpi_sys::MPI_Comm,
137+
info: mpi_sys::MPI_Info,
138+
options: Options,
139+
) -> error::Result<FileMut> {
140+
let f = get_ffi_from_path(path);
141+
let mut ncid: nc_type = -1;
142+
unsafe {
143+
error::checked(with_lock(|| {
144+
netcdf_sys::par::nc_create_par(
145+
f.as_ptr().cast(),
146+
options.bits(),
147+
communicator,
148+
info,
149+
&mut ncid,
150+
)
151+
}))?;
152+
}
153+
154+
Ok(FileMut(File(Self { ncid })))
155+
}
156+
108157
#[cfg(feature = "has-mmap")]
109158
pub(crate) fn open_from_memory<'buffer>(
110159
name: Option<&str>,
@@ -225,6 +274,14 @@ impl File {
225274
.unwrap()
226275
.map(Result::unwrap)
227276
}
277+
/// Get the length of a dimension
278+
pub fn dimension_len(&self, name: &str) -> Option<usize> {
279+
let (ncid, name) =
280+
super::group::try_get_parent_ncid_and_stem(self.ncid(), name).unwrap()?;
281+
super::dimension::dimension_from_name(ncid, name)
282+
.unwrap()
283+
.map(|x| x.len())
284+
}
228285

229286
/// Get a group
230287
///
@@ -440,6 +497,16 @@ impl FileMut {
440497
let Self(File(file)) = self;
441498
file.close()
442499
}
500+
501+
/// Open the file for new definitions
502+
pub fn redef(&mut self) -> error::Result<()> {
503+
error::checked(with_lock(|| unsafe { netcdf_sys::nc_redef(self.ncid()) }))
504+
}
505+
506+
/// Close the file for new definitions
507+
pub fn enddef(&mut self) -> error::Result<()> {
508+
error::checked(with_lock(|| unsafe { netcdf_sys::nc_enddef(self.ncid()) }))
509+
}
443510
}
444511

445512
#[cfg(feature = "has-mmap")]

netcdf/src/lib.rs

+30
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ pub(crate) mod error;
131131
pub(crate) mod extent;
132132
pub(crate) mod file;
133133
pub(crate) mod group;
134+
#[cfg(feature = "mpi")]
135+
pub(crate) mod par;
134136
pub(crate) mod putget;
135137
#[cfg(feature = "4.9.2")]
136138
pub mod rc;
@@ -170,6 +172,20 @@ where
170172
RawFile::create_with(name.as_ref(), options)
171173
}
172174

175+
/// Open a `netCDF` file in create and parallel mode with the given options
176+
#[cfg(feature = "mpi")]
177+
pub fn create_par_with<P>(
178+
name: P,
179+
communicator: mpi_sys::MPI_Comm,
180+
info: mpi_sys::MPI_Info,
181+
options: Options,
182+
) -> error::Result<FileMut>
183+
where
184+
P: AsRef<std::path::Path>,
185+
{
186+
RawFile::create_par_with(name.as_ref(), communicator, info, options)
187+
}
188+
173189
/// Open a `netCDF` file in append mode
174190
pub fn append<P>(name: P) -> error::Result<FileMut>
175191
where
@@ -194,6 +210,20 @@ where
194210
open_with(name, Options::default())
195211
}
196212

213+
/// Open in parallel mode
214+
#[cfg(feature = "mpi")]
215+
pub fn open_par_with<P>(
216+
name: P,
217+
communicator: mpi_sys::MPI_Comm,
218+
info: mpi_sys::MPI_Info,
219+
options: Options,
220+
) -> error::Result<File>
221+
where
222+
P: AsRef<std::path::Path>,
223+
{
224+
RawFile::open_par_with(name.as_ref(), communicator, info, options)
225+
}
226+
197227
/// Open a `netCDF` file in read mode with the given options
198228
pub fn open_with<P>(name: P, options: Options) -> error::Result<File>
199229
where

0 commit comments

Comments
 (0)