From 18023f36f9c5dc9184fd0ca6a29a97305d07cdff Mon Sep 17 00:00:00 2001 From: lfbrehm <97600985+lfbrehm@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:45:00 +0100 Subject: [PATCH] refactor: Updated dockerfiles, workflows and config files to new folder structure --- .../data_proxy/.dockerignore => .dockerignore | 0 .github/workflows/push.yml | 40 ++++++++++--- Cargo.lock | 60 ++++++++++++------- Cargo.toml | 2 +- components/data_proxy/Dockerfile | 11 ++-- .env => components/server/.env | 0 Dockerfile => components/server/Dockerfile | 7 ++- components/server/tests/common/init.rs | 10 ++-- 8 files changed, 86 insertions(+), 44 deletions(-) rename components/data_proxy/.dockerignore => .dockerignore (100%) rename .env => components/server/.env (100%) rename Dockerfile => components/server/Dockerfile (72%) diff --git a/components/data_proxy/.dockerignore b/.dockerignore similarity index 100% rename from components/data_proxy/.dockerignore rename to .dockerignore diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e0ec3b5e..2c827447 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -13,7 +13,8 @@ on: env: # TODO: Change variable to your image's name. - IMAGE_NAME: aruna-server + IMAGE_NAME_SERVER: aruna-server + IMAGE_NAME_PROXY: aruna-data-proxy jobs: push: @@ -24,11 +25,9 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive - - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME - - - - name: Push image to github registry + - name: Build server image + run: docker build . --file ./components/server/Dockerfile --tag $IMAGE_NAME_SERVER + - name: Push server image to github registry run: | # Strip git ref prefix from version VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') @@ -46,5 +45,30 @@ jobs: HARBOR_IMAGE_ID=harbor.computational.bio.uni-giessen.de/aruna/aruna-server - docker tag $IMAGE_NAME $HARBOR_IMAGE_ID:$VERSION - docker push $HARBOR_IMAGE_ID:$VERSION \ No newline at end of file + docker tag $IMAGE_NAME_SERVER $HARBOR_IMAGE_ID:$VERSION + docker push $HARBOR_IMAGE_ID:$VERSION + + + - name: Build proxy image + run: docker build . --file ./components/data_proxy/Dockerfile --tag $IMAGE_NAME_PROXY + - name: Push proxy image to harbor registry + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION="${{ github.run_number }}" + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + echo "${{ secrets.HARBORPW }}" | docker login harbor.computational.bio.uni-giessen.de -u '${{ secrets.HARBORUSER }}' --password-stdin + + HARBOR_IMAGE_ID=harbor.computational.bio.uni-giessen.de/aruna/arunadataproxy + + docker tag $IMAGE_NAME_PROXY $HARBOR_IMAGE_ID:$VERSION + docker push $HARBOR_IMAGE_ID:$VERSION + diff --git a/Cargo.lock b/Cargo.lock index 8370a307..de9de7db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,8 +95,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "aruna-rust-api" -version = "2.0.0-beta.10" -source = "git+https://github.com/ArunaStorage/rust-api.git?branch=pre/beta12#f9b93a98d19e263946123a1dcfb117f9a596681e" +version = "2.0.0-beta.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611f685175025ee0ee5eec3cceb00154f99fc2a75f2e909fa7ea5b3fd34d1408" dependencies = [ "prost", "prost-types", @@ -313,9 +314,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b96342ea8948ab9bef3e6234ea97fc32e2d8a88d8fb6a084e52267317f94b6b" +checksum = "4f4084d18094aec9f79d509f4cb6ccf6b613c5037e32f32e74312e52b836e366" dependencies = [ "aws-credential-types", "aws-runtime", @@ -343,9 +344,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273fa47dafc9ef14c2c074ddddbea4561ff01b7f68d5091c0e9737ced605c01d" +checksum = "fa8587ae17c8e967e4b05a62d495be2fb7701bec52a97f7acfe8a29f938384c8" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -355,9 +356,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e38bab716c8bf07da24be07ecc02e0f5656ce8f30a891322ecdcb202f943b85" +checksum = "b13dc54b4b49f8288532334bba8f87386a40571c47c37b1304979b556dc613c8" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -379,10 +380,11 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.17.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d35d39379445970fc3e4ddf7559fff2c32935ce0b279f9cb27080d6b7c6d94" +checksum = "a297c16c5c2ede4f8d502c70081631e373940634086f90d199ae2e576bd0925d" dependencies = [ + "ahash", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -397,20 +399,25 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", + "fastrand 2.0.1", + "hex", + "hmac", "http 0.2.12", "http-body", + "lru", "once_cell", "percent-encoding", "regex-lite", + "sha2", "tracing", "url", ] [[package]] name = "aws-sdk-sso" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84bd3925a17c9adbf6ec65d52104a44a09629d8f70290542beeee69a95aee7f" +checksum = "c5cc34f5925899739a3f125bd3f7d37d081234a3df218feb9c9d337fd4c70e72" dependencies = [ "aws-credential-types", "aws-runtime", @@ -430,9 +437,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c2dae39e997f58bc4d6292e6244b26ba630c01ab671b6f9f44309de3eb80ab8" +checksum = "7327cddd32b1a6f2aaeaadb1336b671a7975e96a999d3b1bcf5aa47932dc6ddb" dependencies = [ "aws-credential-types", "aws-runtime", @@ -452,9 +459,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17fd9a53869fee17cea77e352084e1aa71e2c5e323d974c13a9c2bcfd9544c7f" +checksum = "6c11981cdb80e8e205e22beb6630a8bdec380a1256bd29efaab34aaebd07cfb9" dependencies = [ "aws-credential-types", "aws-runtime", @@ -475,9 +482,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.1.7" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ada00a4645d7d89f296fe0ddbc3fe3554f03035937c849a05d37ddffc1f29a1" +checksum = "11d6f29688a4be9895c0ba8bef861ad0c0dac5c15e9618b9b7a6c233990fc263" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -661,9 +668,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07c63521aa1ea9a9f92a701f1a08ce3fd20b46c6efc0d5c8947c1fd879e3df1" +checksum = "0dbf2f3da841a8930f159163175cf6a3d16ddde517c1b0fba7aa776822800f40" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -2477,6 +2484,15 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "lru" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +dependencies = [ + "hashbrown 0.14.3", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -4573,9 +4589,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index 64c12b6f..59dba420 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] ahash = "0.8.11" anyhow = "1.0.81" -aruna-rust-api = { git = "https://github.com/ArunaStorage/rust-api.git", branch = "pre/beta12" } +aruna-rust-api = "2.0.0-beta.12" async-channel = "2.2.0" async-trait = "0.1.77" aws-config = "1.1.7" diff --git a/components/data_proxy/Dockerfile b/components/data_proxy/Dockerfile index f40acc04..ec7c981c 100644 --- a/components/data_proxy/Dockerfile +++ b/components/data_proxy/Dockerfile @@ -4,16 +4,17 @@ WORKDIR /build RUN apk update RUN apk upgrade ENV RUSTFLAGS="-C target-feature=-crt-static" +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true RUN apk add llvm cmake gcc ca-certificates libc-dev pkgconfig openssl-dev protoc protobuf-dev protobuf-dev libpq-dev musl-dev git COPY . . -RUN cargo build --release +RUN cargo build --release -p data_proxy FROM alpine:3.18 WORKDIR /run RUN apk update RUN apk upgrade RUN apk add libgcc gcompat ca-certificates libpq -COPY --from=builder /build/target/release/aos_data_proxy . -COPY --from=builder /build/.env . -COPY --from=builder /build/src/database/schema.sql . -CMD [ "/run/aos_data_proxy" ] \ No newline at end of file +COPY --from=builder /build/target/release/data_proxy . +COPY --from=builder /build/components/data_proxy/.env . +COPY --from=builder /build/components/data_proxy/src/database/schema.sql . +CMD [ "/run/data_proxy" ] \ No newline at end of file diff --git a/.env b/components/server/.env similarity index 100% rename from .env rename to components/server/.env diff --git a/Dockerfile b/components/server/Dockerfile similarity index 72% rename from Dockerfile rename to components/server/Dockerfile index c8355ec4..06b45bc2 100644 --- a/Dockerfile +++ b/components/server/Dockerfile @@ -4,9 +4,10 @@ WORKDIR /build RUN apk update RUN apk upgrade ENV RUSTFLAGS="-C target-feature=-crt-static" +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true RUN apk add llvm cmake gcc ca-certificates libc-dev pkgconfig openssl-dev protoc protobuf-dev protobuf-dev libpq-dev musl-dev git COPY . . -RUN cargo build --release +RUN cargo build --release -p aruna_server FROM rust:1-alpine WORKDIR /run @@ -14,7 +15,7 @@ RUN apk update RUN apk upgrade RUN apk add llvm cmake gcc ca-certificates libc-dev pkgconfig openssl-dev protoc protobuf-dev libpq-dev musl-dev git COPY --from=builder /build/target/release/aruna_server . -COPY --from=builder /build/.env . -COPY --from=builder /build/src/database/schema.sql . +COPY --from=builder /build/components/server/.env . +COPY --from=builder /build/components/server/src/database/schema.sql . CMD [ "/run/aruna_server" ] diff --git a/components/server/tests/common/init.rs b/components/server/tests/common/init.rs index 8fe1f5c9..2018aa22 100644 --- a/components/server/tests/common/init.rs +++ b/components/server/tests/common/init.rs @@ -45,7 +45,7 @@ pub struct ServiceBlock { #[allow(dead_code)] pub async fn init_database() -> Arc { // Load env - dotenvy::from_filename(".env").unwrap(); + dotenvy::from_filename("../../.env").unwrap(); // Init database connection let db = Database::new( @@ -82,7 +82,7 @@ pub async fn init_cache(db: Arc, sync: bool) -> Arc { #[allow(dead_code)] pub async fn init_nats_client() -> Arc { // Load env - dotenvy::from_filename(".env").unwrap(); + dotenvy::from_filename("../../.env").unwrap(); // Init NatsIoHandler let client = async_nats::connect(dotenvy::var("NATS_HOST").unwrap()) @@ -99,7 +99,7 @@ pub async fn init_nats_client() -> Arc { #[allow(dead_code)] pub async fn init_search_client() -> Arc { // Load env - dotenvy::from_filename(".env").unwrap(); + dotenvy::from_filename("../../.env").unwrap(); // Init MeilisearchClient let meilisearch_client = MeilisearchClient::new( @@ -182,7 +182,7 @@ pub async fn init_permission_handler( #[allow(dead_code)] pub async fn init_endpoint_service() -> EndpointServiceImpl { // Load env - dotenvy::from_filename(".env").unwrap(); + dotenvy::from_filename("../../.env").unwrap(); // Init database connection let db_conn = init_database().await; @@ -240,7 +240,7 @@ pub async fn init_endpoint_service() -> EndpointServiceImpl { #[allow(dead_code)] pub async fn init_project_service() -> ProjectServiceImpl { // Load env - dotenvy::from_filename(".env").unwrap(); + dotenvy::from_filename("../../.env").unwrap(); // Init database connection let db_conn = init_database().await;