Feat/readme

.gitignore

@@ -1,9 +1,3 @@
-deploy/*
-work/*
-config
-postrun.sh
-SKIP
-SKIP_IMAGES
-.pc
-*-pc
-apt-cacher-ng/
+*.secret
+work/
+deploy/

README.md

@@ -46,6 +46,28 @@ Upon execution, `build.sh` will source the file `config` in the current
 working directory.  This bash shell fragment is intended to set needed
 environment variables.
 
+You can use your own configs with the command line switch 
+`sudo ./build.sh -c config.path.secret`. If the suffix is `.secret` 
+it'll be ignored by git, via .gitignore.
+
+A sample config is as follows:
+
+```bash
+. config
+
+IMG_NAME="${IMG_NAME}-sample"
+
+TARGET_HOSTNAME=sth-pi
+FIRST_USER_NAME=pi
+ENABLE_SSH=1
+# PUBKEY_SSH_FIRST_USER="<your ssh key>"
+PUBKEY_ONLY_SSH=1
+
+WPA_ESSID="<ESSID>"
+WPA_PASSWORD="<password>"
+WPA_COUNTRY=PL
+```
+
 The following environment variables are supported:
 
  * `IMG_NAME` **required** (Default: unset)

README2.md

@@ -0,0 +1,73 @@
+## Preparing the environment for image building
+
+Linux will be required to build the image, so WSL is recommended for windows users.
+
+
+## Dependencies
+The whole process need to be started with the installation of all required dependecies:
+
+
+```bash
+apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
+dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file git curl bc \
+qemu-utils kpartx gpg pigz
+```
+
+The file depends contains a list of tools needed. The format of this package is <tool>[:<debian-package>].
+
+
+## Getting started with building your images
+
+Getting started is as simple as cloning this scramjet's fork on your build machine. You can do so with:
+
+```bash
+git clone [email protected]:scramjetorg/pi-gen.git
+```
+
+We then proceed to build the image. The whole process is done with the command, this may take a while:
+
+
+
+```bash
+./build.sh -c config 
+```
+
+## Burning disc image 
+After the successful build, "deploy" folder will appear with the images.
+
+Now use the [Imager](https://www.raspberrypi.com/software/) and under "Select OS" select "Custom OS" and then select the "<date>-rasppi-os-sth.img" file from the deploy folder you created.
+Also, remember to set your network access for the device in the imager options.
+
+Once the process is complete, two partitions should appear on your drive - boot and deploy.
+
+## Config self-hosted-hub
+
+Go to the scramjet cloud platform panel page and generate a configuration file for self-hosted-hub. Copy its contents and paste it into the file contained in the directory deploy/config/sth-config.json
+
+![1](./assets/1.1.png)
+![2](./assets/1.2.png)
+![3](./assets/1.3.png)
+![4](./assets/1.4.png)
+
+
+Load the card with the configured image into your device and connect it to the power supply. After a while, a new self-hosted-hub should appear in your panel.
+
+
+Now to control the hub using the command line, you need to install "cli" and generate the appropriate config, which will allow access to the hub.
+
+![1 1](./assets/2.1.png)
+![2 1](./assets/2.2.png)
+![3 1](./assets/2.3.png)
+
+
+
+
+
+
+
+## :boom: Troubleshooting :boom:: 
+1. WSL, there may be a problem with WSL, because of the lack of the certain function. This problem is often associated with the "HyperV".
+So to fix it, there is a need to enable HyperV in "Turn Windows features on or off" section.
+
+2. After burning the image on SD card there may be a problem with accessing the partition with the "conf" folder. To solve this problem, there is a need
+to add a new letter for the partition of the disk in "Disk management"

build.sh

@@ -242,6 +242,10 @@ export TIMEZONE_DEFAULT="${TIMEZONE_DEFAULT:-Europe/London}"
 
 export GIT_HASH=${GIT_HASH:-"$(git rev-parse HEAD)"}
 
+export WORK_PARTITION_SIZE=${WORK_PARTITION_SIZE:-8192}
+export DEPLOY_PARTITION_SIZE=${DEPLOY_PARTITION_SIZE:-4096}
+export ROOT_PART_EXTRA_SPACE=${ROOT_PART_EXTRA_SPACE:-8192}
+
 export PUBKEY_SSH_FIRST_USER
 
 export CLEAN

config

@@ -0,0 +1,7 @@
+IMG_NAME="rasppi-os"
+STAGE_LIST=${STAGE_LIST:-"stage0 stage1 stage2 stage99-sth"}
+
+WORK_PARTITION_SIZE=8192
+DEPLOY_PARTITION_SIZE=1024
+
+DEPLOY_COMPRESSION=none

export-image/04-set-partuuid/00-run.sh

@@ -7,10 +7,14 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	IMGID="$(dd if="${IMG_FILE}" skip=440 bs=1 count=4 2>/dev/null | xxd -e | cut -f 2 -d' ')"
 
 	BOOT_PARTUUID="${IMGID}-01"
-	ROOT_PARTUUID="${IMGID}-02"
+	ROOT_PARTUUID="${IMGID}-04"
+	DEPLOY_PARTUUID="${IMGID}-02"
+	WORK_PARTUUID="${IMGID}-03"
 
 	sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+	sed -i "s/DEPLOYDEV/PARTUUID=${DEPLOY_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+	sed -i "s/WORKDEV/PARTUUID=${WORK_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 
 	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
 

export-image/prerun.sh

@@ -10,29 +10,48 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	rm -rf "${ROOTFS_DIR}"
 	mkdir -p "${ROOTFS_DIR}"
 
-	BOOT_SIZE="$((256 * 1024 * 1024))"
+	MEGABYTE="$((1024 * 1024))"
+
+	BOOT_SIZE="$((256 * $MEGABYTE))"
 	ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
+	WORK_SIZE="$(($WORK_PARTITION_SIZE * $MEGABYTE))"
+	DEPLOY_SIZE="$(($DEPLOY_PARTITION_SIZE * $MEGABYTE))"
 
 	# All partition sizes and starts will be aligned to this size
 	ALIGN="$((4 * 1024 * 1024))"
 	# Add this much space to the calculated file size. This allows for
 	# some overhead (since actual space usage is usually rounded up to the
 	# filesystem block size) and gives some free space on the resulting
 	# image.
-	ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + 200 * 1024 * 1024) / 1" | bc)"
+	ROOT_MARGIN="$(echo "($ROOT_SIZE * 0.2 + $ROOT_PART_EXTRA_SPACE * ${MEGABYTE}) / 1" | bc)"
 
 	BOOT_PART_START=$((ALIGN))
 	BOOT_PART_SIZE=$(((BOOT_SIZE + ALIGN - 1) / ALIGN * ALIGN))
-	ROOT_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
+	DEPLOY_PART_START=$((BOOT_PART_START + BOOT_PART_SIZE))
+	DEPLOY_PART_SIZE=$(((DEPLOY_SIZE + ALIGN  - 1) / ALIGN * ALIGN))
+	WORK_PART_START=$((DEPLOY_PART_START + DEPLOY_PART_SIZE))
+	WORK_PART_SIZE=$(((WORK_SIZE + ALIGN  - 1) / ALIGN * ALIGN))
+	ROOT_PART_START=$((WORK_PART_START + WORK_PART_SIZE))
 	ROOT_PART_SIZE=$(((ROOT_SIZE + ROOT_MARGIN + ALIGN  - 1) / ALIGN * ALIGN))
-	IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + ROOT_PART_SIZE))
 
+	IMG_SIZE=$((BOOT_PART_START + BOOT_PART_SIZE + DEPLOY_PART_SIZE + WORK_PART_SIZE + ROOT_PART_SIZE))
+
+	echo Creating paritions:
+
 	truncate -s "${IMG_SIZE}" "${IMG_FILE}"
 
 	parted --script "${IMG_FILE}" mklabel msdos
+	echo BOOT: start="$(($BOOT_PART_START / $MEGABYTE))"MB size="$(( (BOOT_PART_SIZE - 1 ) / $MEGABYTE))"MB
 	parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
+	echo DEPLOY: start="$(($DEPLOY_PART_START / $MEGABYTE))"MB size="$(( (DEPLOY_PART_SIZE - 1 ) / $MEGABYTE))"MB
+	parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${DEPLOY_PART_START}" "$((DEPLOY_PART_START + DEPLOY_PART_SIZE - 1))"
+	echo WORK: start="$(($WORK_PART_START / $MEGABYTE))"MB size="$(( (WORK_PART_SIZE - 1 ) / $MEGABYTE))"MB
+	parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${WORK_PART_START}" "$((WORK_PART_START + WORK_PART_SIZE - 1))"
+	echo ROOT: start="$(($ROOT_PART_START / $MEGABYTE))"MB size="$(( (ROOT_PART_SIZE - 1 ) / $MEGABYTE))"MB
 	parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
 
+	echo TOTAL IMAGE SIZE: $(($IMG_SIZE / $MEGABYTE))MB
+
 	echo "Creating loop device..."
 	cnt=0
 	until LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do
@@ -47,21 +66,37 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	done
 
 	BOOT_DEV="${LOOP_DEV}p1"
-	ROOT_DEV="${LOOP_DEV}p2"
+	ROOT_DEV="${LOOP_DEV}p4"
+	DEPLOY_DEV="${LOOP_DEV}p2"
+	WORK_DEV="${LOOP_DEV}p3"
 
 	ROOT_FEATURES="^huge_file"
 	for FEATURE in 64bit; do
 	if grep -q "$FEATURE" /etc/mke2fs.conf; then
 		ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
 	fi
 	done
+
+	echo "Formatting filesystems..."
 	mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null
+	mkdosfs -n deploy -F 32 -s 4 -v "$DEPLOY_DEV" > /dev/null
+	mkfs.ext4 -L work -O "$ROOT_FEATURES" "$WORK_DEV" > /dev/null
 	mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
 
 	mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
 	mkdir -p "${ROOTFS_DIR}/boot"
 	mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
-
-	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
+	
+	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot --exclude /srv --exclude /opt/sth/deploy "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
 	rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
+
+	if [[ -d "${EXPORT_ROOTFS_DIR}/srv/sth" ]]; then
+		mkdir "${ROOTFS_DIR}/srv" 
+		mkdir "${ROOTFS_DIR}/opt/sth/deploy"
+
+		mount -v "$WORK_DEV" "${ROOTFS_DIR}/srv" -t ext4
+		mount -v "$DEPLOY_DEV" "${ROOTFS_DIR}/opt/sth/deploy" -t vfat
+		rsync -artx "${EXPORT_ROOTFS_DIR}/srv/" "${ROOTFS_DIR}/srv"
+		rsync -rtx "${EXPORT_ROOTFS_DIR}/opt/sth/deploy/" "${ROOTFS_DIR}/opt/sth/deploy"
+	fi
 fi

scripts/qcow2_handling

@@ -225,13 +225,21 @@ function make_bootable_image() {
 	mkdir -p $MOUNTROOT
 
 	MOUNTPT=$MOUNTROOT
-	PARTITION=2
+	PARTITION=4
 	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
 
 	MOUNTPT=$MOUNTROOT/boot
 	PARTITION=1
 	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
 
+	MOUNTPT=$MOUNTROOT/opt/sth/deploy
+	PARTITION=2
+	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
+
+	MOUNTPT=$MOUNTROOT/srv/sth
+	PARTITION=3
+	mount "${EXPORT_IMAGE}" "$MOUNTPT" -o loop,offset=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*start=[ ]*//' | sed 's/,.*//'` * 512 ],sizelimit=$[ `/sbin/sfdisk -d "${EXPORT_IMAGE}" | grep "start=" | head -n $PARTITION | tail -n1 | sed 's/.*size=[ ]*//' | sed 's/,.*//'` * 512 ] || exit 1
+
 	if [ ! -d "${MOUNTROOT}/root" ]; then
 		echo "Image damaged or not mounted. Exit."
 		exit 1
@@ -242,6 +250,8 @@ function make_bootable_image() {
 		echo "Set UUIDs to make it bootable"
 		sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
 		sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/etc/fstab"
+		sed -i "s/DEPLOYDEV/PARTUUID=${DEPLOY_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
+		sed -i "s/WORKDEV/PARTUUID=${WORK_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 		sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${MOUNTROOT}/boot/cmdline.txt"
 	fi
 

stage1/01-sys-tweaks/files/fstab

@@ -1,3 +1,5 @@
-proc            /proc           proc    defaults          0       0
-BOOTDEV  /boot           vfat    defaults          0       2
-ROOTDEV  /               ext4    defaults,noatime  0       1
+proc       /proc           proc    defaults          0       0
+BOOTDEV    /boot           vfat    defaults          0       2
+ROOTDEV    /               ext4    defaults,noatime  0       1
+WORKDEV    /srv            ext4    defaults,noatime  0       1
+DEPLOYDEV  /opt/sth/deploy vfat    defaults,noatime  0       1

stage99-sth/01-setup-user/00-run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+echo "## Creating STH user"
+
+on_chroot << EOF
+adduser sth --system --home /usr/lib/sth
+addgroup sth --system
+
+# Add them to groups
+usermod -aG sudo sth
+usermod -aG gpio sth
+
+# Create a sth-owned folder to put the app in
+# You call this whatever you like, probably the name of your application
+mkdir -p /usr/lib/sth
+EOF

stage99-sth/02-install-node/00-run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+NODE_VERSION=18
+
+echo "Installing node.js $NODE_VERSION"
+
+on_chroot << EOF
+    curl -sL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash -
+    sudo apt-get install -y nodejs jq python3-pip python-is-python3
+EOF

stage99-sth/03-install-sth/00-run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+on_chroot << EOF
+    npm i -g @scramjet/sth
+    pip install pyee==9.0.4 scramjet-framework-py
+EOF

stage99-sth/04-setup-systemd/00-run.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+install -d "${ROOTFS_DIR}/srv/sth"
+install -d "${ROOTFS_DIR}/var/log/sth"
+install -d "${ROOTFS_DIR}/opt/sth/deploy"
+install -d "${ROOTFS_DIR}/opt/sth/deploy/conf"
+install -d "${ROOTFS_DIR}/opt/sth/deploy/sequences"
+
+install -m 755 files/start-sth               "${ROOTFS_DIR}/usr/bin/start-sth"
+install -d                                   "${ROOTFS_DIR}/etc/sth"
+install -d                                   "${ROOTFS_DIR}/etc/sth/sequences"
+install -m 644 files/sth-config              "${ROOTFS_DIR}/etc/sth/"
+install -m 644 files/sth.service             "${ROOTFS_DIR}/etc/systemd/system/"
+install -m 644 files/sth-config-deploy.json  "${ROOTFS_DIR}/opt/sth/deploy/conf/sth-config.json"
+
+on_chroot << EOF
+chown -R sth:sth /usr/lib/sth
+chown -R sth:sth /srv/sth
+chown -R sth:sth /var/log/sth
+systemctl enable sth
+EOF