Skip to content

Commit

Permalink
Merge pull request #15 from sameeul/dynamic_pyr_API_update
Browse files Browse the repository at this point in the history 
PyramidView API Update
  • Loading branch information
@sameeul
sameeul authored Jul 10, 2024
2 parents 1d044cc + 3a89058 commit f515756
Show file tree
Hide file tree
Showing 12 changed files with 122 additions and 205 deletions.
8 changes: 5 additions & 3 deletions .github/workflows/publish_pypi.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ jobs:
bash ci-utils/install_prereq_linux.sh &&
mkdir -p /tmp/argolid_bld &&
cp -r local_install /tmp/argolid_bld
CIBW_BEFORE_ALL_LINUX: yum -y install wget &&
wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 &&
CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 &&
tar -xjf nasm-2.15.05.tar.bz2 &&
cd nasm-2.15.05 &&
./configure &&
Expand All @@ -73,7 +72,10 @@ jobs:
CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja"
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
CIBW_ARCHS: ${{ matrix.cibw_archs }}
CIBW_BEFORE_TEST_LINUX: yum -y install maven java
CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo &&
yum -y install maven java
CIBW_TEST_REQUIRES: bfio tensorstore numpy==1.24.0
CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v

Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/wheel_build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,7 @@ jobs:
bash ci-utils/install_prereq_linux.sh &&
mkdir -p /tmp/argolid_bld &&
cp -r local_install /tmp/argolid_bld
CIBW_BEFORE_ALL_LINUX: yum -y install wget &&
wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 &&
CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 &&
tar -xjf nasm-2.15.05.tar.bz2 &&
cd nasm-2.15.05 &&
./configure &&
Expand All @@ -71,7 +70,10 @@ jobs:
CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja"
CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
CIBW_ARCHS: ${{ matrix.cibw_archs }}
CIBW_BEFORE_TEST_LINUX: yum -y install maven java
CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo &&
yum -y install maven java
CIBW_TEST_REQUIRES: bfio tensorstore numpy==1.24.0
CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v

Expand Down
4 changes: 3 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ find_package(filepattern REQUIRED)
find_package(Threads QUIET)
if (Threads_FOUND)
if (CMAKE_USE_PTHREADS_INIT)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
endif()
endif (CMAKE_USE_PTHREADS_INIT)
list(APPEND Build_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
else ()
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import os
import re
import sys
import sysconfig
import versioneer
import platform
import subprocess
Expand Down Expand Up @@ -102,6 +101,7 @@ def build_extension(self, ext):
package_dir={"": "src/python"},
ext_modules=[CMakeExtension("argolid/libargolid")],
test_suite="tests",
install_requires=["pydantic"],
zip_safe=False,
python_requires=">=3.8",
)
4 changes: 2 additions & 2 deletions src/chunked_base_to_pyr_gen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ void ChunkedBaseToPyramid::CreatePyramidImages( const std::string& input_chunked
int base_level_key,
int min_dim,
VisType v,
std::unordered_map<std::int64_t, DSType>& channel_ds_config,
const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
BS::thread_pool& th_pool)
{
int resolution = 1; // this gets doubled in each level up
Expand Down Expand Up @@ -121,7 +121,7 @@ template <typename T>
void ChunkedBaseToPyramid::WriteDownsampledImage( const std::string& input_file, const std::string& input_scale_key,
const std::string& output_file, const std::string& output_scale_key,
int resolution, VisType v,
std::unordered_map<std::int64_t, DSType>& channel_ds_config,
const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
BS::thread_pool& th_pool)
{
auto [x_dim, y_dim, c_dim, num_dims] = GetZarrParams(v);
Expand Down
4 changes: 2 additions & 2 deletions src/chunked_base_to_pyr_gen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ class ChunkedBaseToPyramid{
int base_scale_key,
int min_dim,
VisType v,
std::unordered_map<std::int64_t, DSType>& channel_ds_config,
const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
BS::thread_pool& th_pool);

private:
template<typename T>
void WriteDownsampledImage( const std::string& input_file, const std::string& input_scale_key,
const std::string& output_file, const std::string& output_scale_key,
int resolution, VisType v,
std::unordered_map<std::int64_t, DSType>& channel_ds_config,
const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
BS::thread_pool& th_pool);
};
} // ns argolid
186 changes: 31 additions & 155 deletions src/pyramid_view.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,24 +25,21 @@
#include "tensorstore/kvstore/kvstore.h"
#include "tensorstore/open.h"
#include "filepattern/filepattern.h"
#include <nlohmann/json.hpp>

#include "pyramid_view.h"
#include "chunked_base_to_pyr_gen.h"
#include "utilities.h"
#include "pugixml.hpp"
#include <plog/Log.h>
#include "plog/Initializers/RollingFileInitializer.h"
#include <nlohmann/json.hpp>
using json = nlohmann::json;
namespace fs = std::filesystem;

using::tensorstore::Context;
using::tensorstore::internal_zarr::ChooseBaseDType;

namespace argolid {

void PyramidView::AssembleBaseLevel(VisType v) {
void PyramidView::AssembleBaseLevel(VisType v, const image_map& coordinate_map, const std::string& zarr_array_path) {
if (v!=VisType::NG_Zarr && v!=VisType::Viv) {
PLOG_INFO << "Unsupported Pyramid type requested";
return;
Expand All @@ -51,30 +48,30 @@ namespace argolid {
int grid_x_max = 0, grid_y_max = 0, grid_c_max = 0;

int img_count = 0;
for (const auto & [name, location]: base_image_map) {
for (const auto & [name, location]: coordinate_map) {
const auto[gx, gy, gc] = location;
gc > grid_c_max ? grid_c_max = gc : grid_c_max = grid_c_max;
gx > grid_x_max ? grid_x_max = gx : grid_x_max = grid_x_max;
gy > grid_y_max ? grid_y_max = gy : grid_y_max = grid_y_max;
++img_count;
}
PLOG_INFO << "Total images found: " << img_count << std::endl;
PLOG_DEBUG << "Total images found: " << img_count << std::endl;
auto t1 = std::chrono::high_resolution_clock::now();
auto [x_dim, y_dim, c_dim, num_dims] = GetZarrParams(v);

ImageInfo whole_image;

if (img_count != 0) {
size_t write_failed_count = 0;
const auto & sample_tiff_file = image_coll_path + "/" + base_image_map.begin() -> first;
const auto & sample_tiff_file = image_coll_path + "/" + coordinate_map.begin() -> first;
TENSORSTORE_CHECK_OK_AND_ASSIGN(auto test_source, tensorstore::Open(
GetOmeTiffSpecToRead(sample_tiff_file),
tensorstore::OpenMode::open,
tensorstore::ReadWriteMode::read).result());
auto test_image_shape = test_source.domain().shape();

whole_image._chunk_size_x = test_image_shape[4];
whole_image._chunk_size_y = test_image_shape[3];
whole_image._chunk_size_x = test_image_shape[4] + 2*x_spacing;
whole_image._chunk_size_y = test_image_shape[3] + 2*y_spacing;
whole_image._full_image_width = (grid_x_max + 1) * whole_image._chunk_size_x;
whole_image._full_image_height = (grid_y_max + 1) * whole_image._chunk_size_y;
whole_image._num_channels = grid_c_max + 1;
Expand All @@ -88,8 +85,8 @@ namespace argolid {
whole_image._data_type = test_source.dtype().name();
new_image_shape[c_dim] = whole_image._num_channels;

auto output_spec = [&test_source, &new_image_shape, &chunk_shape, this]() {
return GetZarrSpecToWrite(base_zarr_path, new_image_shape, chunk_shape, ChooseBaseDType(test_source.dtype()).value().encoded_dtype);
auto output_spec = [&test_source, &new_image_shape, &chunk_shape, &zarr_array_path, this]() {
return GetZarrSpecToWrite(zarr_array_path, new_image_shape, chunk_shape, ChooseBaseDType(test_source.dtype()).value().encoded_dtype);
}();

TENSORSTORE_CHECK_OK_AND_ASSIGN(auto dest, tensorstore::Open(
Expand All @@ -99,14 +96,14 @@ namespace argolid {
tensorstore::ReadWriteMode::write).result());

auto t4 = std::chrono::high_resolution_clock::now();
for (const auto & [file_name, location]: base_image_map) {
for (const auto & [file_name, location]: coordinate_map) {
th_pool.push_task([ &dest, file_name=file_name, location=location, x_dim=x_dim, y_dim=y_dim, c_dim=c_dim, v, &whole_image, this]() {

TENSORSTORE_CHECK_OK_AND_ASSIGN(auto source, tensorstore::Open(
GetOmeTiffSpecToRead(image_coll_path + "/" + file_name),
tensorstore::OpenMode::open,
tensorstore::ReadWriteMode::read).result());
PLOG_INFO << "Opening " << file_name;
PLOG_DEBUG << "Opening " << file_name;
auto image_shape = source.domain().shape();
auto image_width = image_shape[4];
auto image_height = image_shape[3];
Expand All @@ -127,12 +124,12 @@ namespace argolid {
tensorstore::IndexTransform < > transform = tensorstore::IdentityTransform(dest.domain());
if (v == VisType::NG_Zarr) {
transform = (std::move(transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y, image_height) |
tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x, image_width)).value();
tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y + y_spacing, image_height) |
tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x + x_spacing, image_width)).value();
} else if (v == VisType::Viv) {
transform = (std::move(transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y, image_height) |
tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x, image_width)).value();
tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y + y_spacing, image_height) |
tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x + x_spacing, image_width)).value();
}
tensorstore::Write(array, dest | transform).value();
});
Expand All @@ -143,165 +140,44 @@ namespace argolid {
base_image = whole_image;
}

void PyramidView::ReAssembleBaseLevelWithNewMap(VisType v, const image_map& m, const std::string& output_path) {

if (v!=VisType::NG_Zarr && v!=VisType::Viv) {
PLOG_INFO << "Unsupported Pyramid type requested";
return;
}

auto [x_dim, y_dim, c_dim, num_dims] = GetZarrParams(v);

auto input_spec = [this]() {
return GetZarrSpecToRead(base_zarr_path);
}();

TENSORSTORE_CHECK_OK_AND_ASSIGN(auto base_store, tensorstore::Open(
input_spec,
tensorstore::OpenMode::open,
tensorstore::ReadWriteMode::read).result());
auto base_image_shape = base_store.domain().shape();
auto read_chunk_shape = base_store.chunk_layout().value().read_chunk_shape();

std::vector < std::int64_t > new_image_shape(num_dims, 1);
std::vector < std::int64_t > chunk_shape(num_dims, 1);

new_image_shape[y_dim] = base_image_shape[y_dim];
new_image_shape[x_dim] = base_image_shape[x_dim];

chunk_shape[y_dim] = read_chunk_shape[y_dim];
chunk_shape[x_dim] = read_chunk_shape[x_dim];

auto open_mode = tensorstore::OpenMode::create;
open_mode = open_mode | tensorstore::OpenMode::delete_existing;
new_image_shape[c_dim] = base_image_shape[c_dim];


auto output_spec = [v, &output_path, &new_image_shape, & chunk_shape, & base_store, this]() {
if (v == VisType::NG_Zarr) {
return GetZarrSpecToWrite(output_path + "/0", new_image_shape, chunk_shape, ChooseBaseDType(base_store.dtype()).value().encoded_dtype);
} else if (v == VisType::Viv) {
return GetZarrSpecToWrite(output_path + "/0", new_image_shape, chunk_shape, ChooseBaseDType(base_store.dtype()).value().encoded_dtype);
}
}();

TENSORSTORE_CHECK_OK_AND_ASSIGN(auto dest, tensorstore::Open(
output_spec,
open_mode,
tensorstore::ReadWriteMode::write).result());

size_t write_failed_count = 0;

for (const auto & [file_name, location]: m) {
// find where to read data from
const auto base_location = [file_name=file_name, this]() -> std::optional < std::tuple < std::uint32_t,
uint32_t, uint32_t >> {
if (auto search = base_image_map.find(file_name); search != base_image_map.end()) {
return std::optional {
search -> second
};
} else {
return std::nullopt;
}
}();

if (!base_location.has_value()) {
continue;
}

th_pool.push_task([ &base_store, &dest, file_name=file_name, location=location, base_location, x_dim=x_dim, y_dim=y_dim, c_dim=c_dim, v, this]() {

const auto & [x_grid_base, y_grid_base, c_grid_base] = base_location.value();

tensorstore::IndexTransform < > read_transform = tensorstore::IdentityTransform(base_store.domain());

if (v == VisType::NG_Zarr) {
read_transform = (std::move(read_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid_base, 1) |
tensorstore::Dims(y_dim).SizedInterval(y_grid_base * base_image._chunk_size_y, base_image._chunk_size_y) |
tensorstore::Dims(x_dim).SizedInterval(x_grid_base * base_image._chunk_size_x, base_image._chunk_size_x)).value();
} else if (v == VisType::Viv) {
read_transform = (std::move(read_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid_base, 1) |
tensorstore::Dims(y_dim).SizedInterval(y_grid_base * base_image._chunk_size_y, base_image._chunk_size_y) |
tensorstore::Dims(x_dim).SizedInterval(x_grid_base * base_image._chunk_size_x, base_image._chunk_size_x)).value();
}

auto array = tensorstore::AllocateArray({
base_image._chunk_size_y,
base_image._chunk_size_x
}, tensorstore::c_order,
tensorstore::value_init, base_store.dtype());

// initiate a read
tensorstore::Read(base_store | read_transform, array).value();

const auto & [x_grid, y_grid, c_grid] = location;

tensorstore::IndexTransform < > write_transform = tensorstore::IdentityTransform(dest.domain());
if (v == VisType::NG_Zarr) {
write_transform = (std::move(write_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
tensorstore::Dims(y_dim).SizedInterval(y_grid * base_image._chunk_size_y, base_image._chunk_size_y) |
tensorstore::Dims(x_dim).SizedInterval(x_grid * base_image._chunk_size_x, base_image._chunk_size_x)).value();
} else if (v == VisType::Viv) {
write_transform = (std::move(write_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
tensorstore::Dims(y_dim).SizedInterval(y_grid * base_image._chunk_size_y, base_image._chunk_size_y) |
tensorstore::Dims(x_dim).SizedInterval(x_grid * base_image._chunk_size_x, base_image._chunk_size_x)).value();
}
tensorstore::Write(array, dest | write_transform).value();
});
}

th_pool.wait_for_tasks();
}

void PyramidView::GeneratePyramid(std::optional<image_map> map,
void PyramidView::GeneratePyramid(const image_map& map,
VisType v,
int min_dim,
std::unordered_map<std::int64_t, DSType>& channel_ds_config)
const std::unordered_map<std::int64_t, DSType>& channel_ds_config)
{
const auto image_dir = pyramid_zarr_path + "/" + image_name +".zarr";
if (fs::exists(image_dir)) fs::remove_all(image_dir);
PLOG_INFO << "GeneratePyramid Start ";
if (v!=VisType::NG_Zarr && v!=VisType::Viv) {
PLOG_INFO << "Unsupported Pyramid type requested";
return;
}
const auto output_zarr_path = [v, this](){


const auto output_zarr_path = [v, &image_dir, this](){
if (v==VisType::Viv){
return pyramid_zarr_path + "/" + image_name +".zarr/data.zarr/0";
return image_dir +"/data.zarr/0";
} else {
return pyramid_zarr_path + "/" + image_name +".zarr/0";
return image_dir +"/0";
}
}();

if (map.has_value()){
ReAssembleBaseLevelWithNewMap(v,map.value(),output_zarr_path);
} else {
// copy base level zarr file
fs::path destination{output_zarr_path+"/0"};
if (!fs::exists(destination)) {
fs::create_directories(destination);
}

// Iterate over files in the source directory
fs::path source{base_zarr_path};
for (const auto& entry : fs::directory_iterator(source)) {
const auto& path = entry.path();
auto destPath = destination / path.filename();

// Copy file
if (fs::is_regular_file(path)) {
fs::copy_file(path, destPath, fs::copy_options::overwrite_existing);
}
}
}
PLOG_INFO << "Starting to generate base layer ";
AssembleBaseLevel(v, map, output_zarr_path+"/0") ;
PLOG_INFO << "Finished generating base layer ";

// generate pyramid
ChunkedBaseToPyramid base_to_pyramid;
int base_level_key = 0;
int max_level = static_cast<int>(ceil(log2(std::max({base_image._full_image_width, base_image._full_image_width}))));
int min_level = static_cast<int>(ceil(log2(min_dim)));
auto max_level_key = max_level-min_level+1;
PLOG_INFO << "Starting to generate pyramid ";
base_to_pyramid.CreatePyramidImages(output_zarr_path, output_zarr_path, base_level_key, min_dim, v, channel_ds_config, th_pool);

PLOG_INFO << "Finished generating pyramid ";

// generate metadata
WriteMultiscaleMetadataForImageCollection(image_name, pyramid_zarr_path, base_level_key, max_level_key, v, base_image);
PLOG_INFO << "GeneratePyramid end ";
}

} // ns argolid
Loading

0 comments on commit f515756

Please sign in to comment.