Use new dl-pipe utility to get resumable piped downloads (#6)

* Use new dl-pipe utility to get resumable piped downloads

* move genesis bin creation and link to init.sh

This is to tolerate .zetacored being mounted on a volume

* fix snapshotter build

* add hash verification

* add procps

* use testnet snapshot for CI

* fix init_completed

* remove VOLUME

* add restart CI

Author: gartnera

.github/workflows/ci.yml

@@ -59,7 +59,7 @@ jobs:
         with:
           platforms: linux/amd64,linux/arm64
           push: true
-          target: base
+          target: snapshotter
           provenance: mode=max
           tags: "${{ env.IMAGE_BASE }}:${{ env.VERSION }}-snapshotter"
           cache-from: type=registry,ref=${{ env.IMAGE_BASE }}:buildcache
@@ -70,7 +70,7 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Start container
-        run: docker run -d --name zetacored -p 8545:8545 -e MONIKER=$(uuidgen) ${{ needs.build.outputs.IMAGE }}
+        run: docker run -d --name zetacored -p 8545:8545 -e MONIKER=$(uuidgen) -e ZETACHAIN_NETWORK=testnet ${{ needs.build.outputs.IMAGE }}
       - name: Wait for healthy
         run: |
           while ! curl -s -f --data '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' -H 'Content-Type: application/json' http://localhost:8545; do
@@ -85,4 +85,23 @@ jobs:
       - name: Dump logs
         if: always()
         run: docker logs zetacored
+      - name: Ensure we can restart
+        run: |
+          docker stop zetacored
+          docker start zetacored
+      - name: Wait for healthy
+        run: |
+          while ! curl -s -f --data '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' -H 'Content-Type: application/json' http://localhost:8545; do
+            if ! docker ps | grep zetacored; then
+              echo "Container stopped?"
+              exit 1
+            fi
+            df -h /
+            echo "waiting for zetacored health"
+            sleep 15
+          done
+      - name: Dump logs (restart)
+        if: always()
+        run: docker logs --since 1m zetacored
+
 

Dockerfile

@@ -2,32 +2,28 @@ FROM golang:1.22.5-bookworm AS base-build
 
 RUN go install cosmossdk.io/tools/cosmovisor/cmd/[email protected]
 RUN go install github.com/hashicorp/go-getter/cmd/[email protected]
+RUN go install github.com/zeta-chain/dl-pipe/cmd/dl-pipe@latest
 
 FROM debian:bookworm AS base
 
-RUN mkdir -p /root/.zetacored/cosmovisor/genesis/bin && \
-    ln -s /root/.zetacored/cosmovisor/genesis /root/.zetacored/cosmovisor/current
-
 ENV PATH=/root/.zetacored/cosmovisor/current/bin/:${PATH}
 
 RUN apt update && \
     apt install -y ca-certificates curl jq && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --from=base-build /go/bin/cosmovisor /go/bin/go-getter /usr/local/bin
+COPY --from=base-build /go/bin/cosmovisor /go/bin/go-getter /go/bin/dl-pipe /usr/local/bin
 
 COPY run.sh init.sh /
 
-VOLUME /root/.zetacored/data/
-
 ENTRYPOINT ["/run.sh"]
 
 FROM base AS snapshotter
 
 ARG TARGETARCH
 
 RUN apt update && \
-    apt install -y rclone && \
+    apt install -y rclone procps && \
     rm -rf /var/lib/apt/lists/*
 
 RUN ARCH=$( [ "$TARGETARCH" = "amd64" ] && echo "x86_64" || echo "$TARGETARCH" ) && \

init.sh

@@ -66,6 +66,14 @@ download_configs() {
 
 install_genesis_zetacored() {
   echo "Installing genesis zetacored"
+
+  # create the genesis bin path and symlink it to the current path
+  genesis_path=".zetacored/cosmovisor/genesis"
+  mkdir -p "$genesis_path"
+  mkdir -p "${genesis_path}/bin"
+  current_path=".zetacored/cosmovisor/current"
+  ln -s "${HOME}/${genesis_path}" "${HOME}/${current_path}"
+
   if [[ -z $ZETACORED_BINARY_URL ]]; then
     max_height=$($CURL "${ZETACHAIN_SNAPSHOT_METADATA_URL}" | jq -r '.snapshots[0].height')
     echo "Getting latest passed upgrade plan before ${max_height}"
@@ -87,9 +95,12 @@ install_genesis_zetacored() {
 }
 
 restore_snapshot() {
-  snapshot_link=$($CURL "${ZETACHAIN_SNAPSHOT_METADATA_URL}" | jq -r '.snapshots[0].link')
+  snapshot=$($CURL "${ZETACHAIN_SNAPSHOT_METADATA_URL}" | jq -r '.snapshots[0]')
+  snapshot_link=$(echo "$snapshot" | jq -r '.link')
+  snapshot_md5=$(echo "$snapshot" | jq -r '.checksums.md5')
   echo "Restoring snapshot from ${snapshot_link}"
-  $CURL "$snapshot_link" | tar x -C $HOME/.zetacored
+  # https://github.com/zeta-chain/dl-pipe
+  dl-pipe -hash "md5:${snapshot_md5}" "$snapshot_link" | tar x -C $HOME/.zetacored
 }
 
 cd $HOME
@@ -100,12 +111,12 @@ if [[ -f /root/init_started && ! -f /root/init_completed ]]; then
 fi
 
 touch /root/init_started
-if [[ ! -f /root/init_complete ]]; then
+if [[ ! -f /root/init_completed ]]; then
   echo "Starting initialization"
   download_configs
   install_genesis_zetacored
   restore_snapshot
-  touch /root/init_complete
+  touch /root/init_completed
 else 
   echo "Initialization already completed"
 fi