Merge pull request #405 from subspace/release-preparation

Release preparation

Author: nazar-pc

Dockerfile-node

@@ -39,10 +39,15 @@ FROM ubuntu:20.04
 
 RUN \
     apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+HEALTHCHECK CMD curl \
+                  -H "Content-Type: application/json" \
+                  -d '{ "id": 1, "jsonrpc": "2.0", "method": "system_health", "params": [] }' \
+                  -f "http://localhost:9933"
+
 COPY --from=0 /code/subspace-node /subspace-node
 
 RUN mkdir /var/subspace && chown nobody:nogroup /var/subspace

crates/sc-consensus-subspace-rpc/src/lib.rs

@@ -557,7 +557,9 @@ where
 
             if let Some(mut sender) = maybe_sender {
                 if let Err(error) = sender.send(()).await {
-                    warn!("Failed to acknowledge archived segment: {error}");
+                    if !error.is_disconnected() {
+                        warn!("Failed to acknowledge archived segment: {error}");
+                    }
                 }
             }
 

crates/subspace-farmer/src/archiving.rs

@@ -171,13 +171,17 @@ impl Archiving {
                             Some(archived_segment) => {
                                 let segment_index = archived_segment.root_block.segment_index();
                                 let (acknowledge_sender, acknowledge_receiver) = oneshot::channel();
+                                // Acknowledge immediately to allow node to continue sync quickly,
+                                // but this will miss some segments in case farmer crashed in the
+                                // meantime. Ideally we'd acknowledge after, but it makes node wait
+                                // for it and the whole process very sequential.
+                                if let Err(error) = client.acknowledge_archived_segment(segment_index).await {
+                                    error!("Failed to send archived segment acknowledgement: {error}");
+                                }
                                 if let Err(error) = archived_segments_sync_sender.send((archived_segment, acknowledge_sender)) {
                                     error!("Failed to send archived segment for plotting: {error}");
                                 }
                                 let _ = acknowledge_receiver.await;
-                                if let Err(error) = client.acknowledge_archived_segment(segment_index).await {
-                                    error!("Failed to send archived segment acknowledgement: {error}");
-                                }
                             },
                             None => {
                                 debug!("Subscription has forcefully closed from node side!");

crates/subspace-farmer/src/bin/subspace-farmer/main.rs

@@ -64,7 +64,12 @@ enum Command {
 }
 
 fn parse_human_readable_size(s: &str) -> Result<u64, std::num::ParseIntError> {
-    const SUFFIXES: &[(&str, u64)] = &[("G", 10u64.pow(9)), ("T", 10u64.pow(12))];
+    const SUFFIXES: &[(&str, u64)] = &[
+        ("G", 10u64.pow(9)),
+        ("GB", 10u64.pow(9)),
+        ("T", 10u64.pow(12)),
+        ("TB", 10u64.pow(12)),
+    ];
 
     SUFFIXES
         .iter()

crates/subspace-node/src/bin/subspace-node.rs

@@ -268,8 +268,11 @@ fn main() -> std::result::Result<(), Error> {
                         primary_chain_node_config,
                         true,
                     )
-                    .map_err(|_| {
-                        sc_service::Error::Other("Failed to build a full subspace node".into())
+                    .map_err(|error| {
+                        sc_service::Error::Other(format!(
+                            "Failed to build a full subspace node: {}",
+                            error
+                        ))
                     })?
                 };
 
@@ -290,10 +293,11 @@ fn main() -> std::result::Result<(), Error> {
                         &secondary_chain_cli,
                         tokio_handle,
                     )
-                    .map_err(|_| {
-                        sc_service::Error::Other(
-                            "Failed to create secondary chain configuration".into(),
-                        )
+                    .map_err(|error| {
+                        sc_service::Error::Other(format!(
+                            "Failed to create secondary chain configuration: {}",
+                            error
+                        ))
                     })?;
 
                     let secondary_chain_full_node_fut = cirrus_node::service::new_full(

docs/farming.md

@@ -1,3 +1,6 @@
+# ⚠️ Living document
+
+**‼️ NOTE: This is a living document reflecting current state of the codebase, make sure to open this page from the [release you want to install](https://github.com/subspace/subspace/releases) and not directly ‼️**
 
 # 👨‍🌾 Getting Started Farming
 
@@ -36,12 +39,11 @@ The address of your account will be necessary at the last step.
 # Copy all of the lines below, they are all part of the same command
 .\NODE_FILE_NAME.exe `
 --chain testnet `
---wasm-execution compiled `
---execution wasm `
---bootnodes "/dns/farm-rpc.subspace.network/tcp/30333/p2p/12D3KooWPjMZuSYj35ehced2MTJFf95upwpHKgKUrFRfHwohzJXr" `
+--execution native `
+--unsafe-pruning `
+--pruning 1024 `
+--keep-blocks 1024 `
 --validator `
---telemetry-url "wss://telemetry.polkadot.io/submit/ 1" `
---telemetry-url "wss://telemetry.subspace.network/submit 1" `
 --name INSERT_YOUR_ID
 ```
 5. You should see something similar in the terminal:
@@ -71,7 +73,8 @@ The address of your account will be necessary at the last step.
 ```PowerShell
 # Replace `FARMER_FILE_NAME.exe` with the name of the node file you downloaded from releases
 # Replace `WALLET_ADDRESS` below with your account address from Polkadot.js wallet
-.\FARMER_FILE_NAME.exe farm --reward-address WALLET_ADDRESS
+# Replace `PLOT_SIZE` with plot size in gigabytes or terabytes, for instance 100G or 2T (but leave at least 10G of disk space for node)
+.\FARMER_FILE_NAME.exe farm --reward-address WALLET_ADDRESS --plot-size PLOT_SIZE
 ```
 
 ## 🐧 Linux Instructions
@@ -87,12 +90,11 @@ The address of your account will be necessary at the last step.
 # Copy all of the lines below, they are all part of the same command
 ./NODE_FILE_NAME \
   --chain testnet \
-  --wasm-execution compiled \
   --execution wasm \
-  --bootnodes "/dns/farm-rpc.subspace.network/tcp/30333/p2p/12D3KooWPjMZuSYj35ehced2MTJFf95upwpHKgKUrFRfHwohzJXr" \
+  --unsafe-pruning \
+  --pruning 1024 \
+  --keep-blocks 1024 \
   --validator \
-  --telemetry-url "wss://telemetry.polkadot.io/submit/ 1" \
-  --telemetry-url "wss://telemetry.subspace.network/submit 1" \
   --name INSERT_YOUR_ID
 ```
 5. You should see something similar in the terminal:
@@ -121,7 +123,8 @@ The address of your account will be necessary at the last step.
 ```bash
 # Replace `FARMER_FILE_NAME` with the name of the node file you downloaded from releases
 # Replace `WALLET_ADDRESS` below with your account address from Polkadot.js wallet
-./FARMER_FILE_NAME farm --reward-address WALLET_ADDRESS
+# Replace `PLOT_SIZE` with plot size in gigabytes or terabytes, for instance 100G or 2T (but leave at least 10G of disk space for node)
+./FARMER_FILE_NAME farm --reward-address WALLET_ADDRESS --plot-size PLOT_SIZE
 ```
 
 ## 🍎 macOS Instructions
@@ -141,12 +144,11 @@ After this, simply repeat the step you prompted for (step 4 or 6). This time, cl
 # Copy all of the lines below, they are all part of the same command
 ./NODE_FILE_NAME \
   --chain testnet \
-  --wasm-execution compiled \
   --execution wasm \
-  --bootnodes "/dns/farm-rpc.subspace.network/tcp/30333/p2p/12D3KooWPjMZuSYj35ehced2MTJFf95upwpHKgKUrFRfHwohzJXr" \
+  --unsafe-pruning \
+  --pruning 1024 \
+  --keep-blocks 1024 \
   --validator \
-  --telemetry-url "wss://telemetry.polkadot.io/submit/ 1" \
-  --telemetry-url "wss://telemetry.subspace.network/submit 1" \
   --name INSERT_YOUR_ID
 ```
 5. You should see something similar in the terminal:
@@ -175,7 +177,8 @@ After this, simply repeat the step you prompted for (step 4 or 6). This time, cl
 ```bash
 # Replace `FARMER_FILE_NAME` with the name of the node file you downloaded from releases
 # Replace `WALLET_ADDRESS` below with your account address from Polkadot.js wallet
-./FARMER_FILE_NAME farm --reward-address WALLET_ADDRESS
+# Replace `PLOT_SIZE` with plot size in gigabytes or terabytes, for instance 100G or 2T (but leave at least 10G of disk space for node)
+./FARMER_FILE_NAME farm --reward-address WALLET_ADDRESS --plot-size PLOT_SIZE
 ```
 7. It may prompt again in here. Refer to the note on step 4.
 
@@ -186,7 +189,7 @@ Create `subspace` directory and `docker-compose.yml` in it with following conten
 version: "3.7"
 services:
   node:
-    # Replace `snapshot-DATE` with latest release (like `snapshot-2022-mar-09`)
+    # Replace `snapshot-DATE` with latest release (like `snapshot-2022-apr-29`)
     image: ghcr.io/subspace/node:snapshot-DATE
     volumes:
 # Instead of specifying volume (which will store data in `/var/lib/docker`), you can
@@ -202,24 +205,29 @@ services:
     command: [
       "--chain", "testnet",
       "--base-path", "/var/subspace",
-      "--wasm-execution", "compiled",
       "--execution", "wasm",
-      "--bootnodes", "/dns/farm-rpc.subspace.network/tcp/30333/p2p/12D3KooWPjMZuSYj35ehced2MTJFf95upwpHKgKUrFRfHwohzJXr",
+      "--unsafe-pruning",
+      "--pruning", "1024",
+      "--keep-blocks", "1024",
       "--port", "30333",
-      "--telemetry-url", "wss://telemetry.polkadot.io/submit/ 1",
-      "--telemetry-url", "wss://telemetry.subspace.network/submit/ 1",
       "--rpc-cors", "all",
       "--rpc-methods", "safe",
       "--unsafe-ws-external",
       "--validator",
 # Replace `INSERT_YOUR_ID` with your node ID (will be shown in telemetry)
       "--name", "INSERT_YOUR_ID"
     ]
+    healthcheck:
+      timeout: 5s
+# If node setup takes longer then expected, you want to increase `interval` and `retries` number.
+      interval: 30s
+      retries: 5
 
   farmer:
     depends_on:
-      - node
-# Replace `snapshot-DATE` with latest release (like `snapshot-2022-mar-09`)
+      node:
+        condition: service_healthy
+# Replace `snapshot-DATE` with latest release (like `snapshot-2022-apr-29`)
     image: ghcr.io/subspace/farmer:snapshot-DATE
 # Un-comment following 2 lines to unlock farmer's RPC
 #    ports:
@@ -237,7 +245,9 @@ services:
       "--node-rpc-url", "ws://node:9944",
       "--ws-server-listen-addr", "0.0.0.0:9955",
 # Replace `WALLET_ADDRESS` with your Polkadot.js wallet address
-      "--reward-address", "WALLET_ADDRESS"
+      "--reward-address", "WALLET_ADDRESS",
+# Replace `PLOT_SIZE` with plot size in gigabytes or terabytes, for instance 100G or 2T (but leave at least 10G of disk space for node)
+      "--plot-size", "PLOT_SIZE"
     ]
 volumes:
   node-data:
@@ -246,10 +256,11 @@ volumes:
 
 After which follow these steps:
 * Now edit created file:
-  1. Replace `snapshot-DATE` with the latest release (not pre-release!) snapshot (like `snapshot-2022-mar-09`)
-  2. Replace `INSERT_YOUR_ID` with desired name that will be shown in telemetry (doesn't impact anything else)
-  3. Replace `WALLET_ADDRESS` with your wallet address
-  4. If you want to store files on a separate disk or customize port, read comments in the file
+  * Replace `snapshot-DATE` with the latest release (not pre-release!) snapshot (like `snapshot-2022-apr-29`)
+  * Replace `INSERT_YOUR_ID` with desired name that will be shown in telemetry (doesn't impact anything else)
+  * Replace `WALLET_ADDRESS` with your wallet address
+  * Replace `PLOT_SIZE` with plot size in gigabytes or terabytes, for instance 100G or 2T (but leave at least 10G of disk space for node)
+  * If you want to store files on a separate disk or customize port, read comments in the file
 * Ensure [Docker](https://www.docker.com/) is installed and running
 * Now go to directory with `docker-compose.yml` and type `docker-compose up -d` to start everything
 
@@ -264,7 +275,6 @@ Visit [Polkadot.js explorer](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Ffarm-
 ## Invalid Solution
 If you are getting `invalid solution` errors (visible on the terminal that Node runs), please follow "Switching to a new snapshot" steps below and start afresh.
 
----
 ## Switching to a new snapshot
 
 ### CLI
@@ -284,18 +294,20 @@ Now follow installation guide.
 
 ### Docker
 
-In case of Docker setup run `docker-compose down` (and manually delete custom directories if you have specified them).
+In case of Docker setup run `docker-compose down -v` (and manually delete custom directories if you have specified them).
 
 Now follow installation guide.
 
 ## Help
 
 There are extra commands and parameters you can use on farmer or node, use the `--help` after any other command to display additional options.
 
-Below are some helpful farmer commands:
+Below are some helpful samples:
 
-- `farm --reward-address WALLET_ADDRESS` : starts background plotting and farming together, farmed testnet coins will be sent to `WALLET_ADDRESS`
-- `wipe` : erases the plot and identity (including plot, commitment, object mappings and identity files)
+- `./FARMER_FILE_NAME farm --custom-path /path/to/data ...` : will store data in `/path/to/data` instead of default location
+- `./FARMER_FILE_NAME wipe --custom-path /path/to/data` : erases everything related to farmer if data were stored in `/path/to/data`
+- `./NODE_FILE_NAME --base-path /path/to/data --chain testnet ...` : start node and store data in `/path/to/data` instead of default location
+- `./NODE_FILE_NAME purge-chain --base-path /path/to/data --chain testnet` : erases data related to the node if data were stored in `/path/to/data`
 
 Examples:
 ```bash
@@ -304,6 +316,20 @@ Examples:
 ./FARMER_FILE_NAME wipe
 ```
 
+## [Advanced] Running an archival node
+
+Instructions above will get you full node (doesn't store the history and state of the whole blockchain, only last 1024
+blocks). If you want to opt in to storing the whole history (archival node), remove following parameters (lines) from
+above instructions before starting your node:
+* `--unsafe-pruning`
+* `--pruning 1024`
+* `--keep-blocks 1024`
+
+Archival node is useful if you run an RPC node and want to support querying older blockchain history.
+
+NOTE: You can't switch between full and archival node without wiping it, so if you need that, follow steps in
+[Switching to a new snapshot](#switching-to-a-new-snapshot) section above.
+
 ## [Advanced] Build from source (Linux)
 
 If you're running unsupported Linux distribution or CPU architecture, you may try to build binaries yourself from source.
@@ -316,17 +342,15 @@ You'll have to have [Rust toolchain](https://rustup.rs/) installed as well as LL
 sudo apt-get install llvm clang
 ```
 
-Now clone the source and build snapshot `snapshot-2022-mar-09` (replace occurrences with the snapshot you want to build):
+Now clone the source and build snapshot `snapshot-2022-apr-29` (replace occurrences with the snapshot you want to build):
 ```bash
 git clone https://github.com/subspace/subspace.git
 cd subspace
-git checkout snapshot-2022-mar-09
-wget -O chain-spec.json https://github.com/subspace/subspace/releases/download/snapshot-2022-mar-09/chain-spec-raw-snapshot-2022-mar-09.json
+git checkout snapshot-2022-apr-29
 cargo build \
     --profile production \
     --bin subspace-node \
-    --bin subspace-farmer \
-    --features=subspace-node/json-chain-spec
+    --bin subspace-farmer
 ```
 
 You'll find two binaries under `target/production` directory once it succeeds, after which refer to instructions above on how to use them.