Merge pull request #6 from ading2210/new-scripts

New scripts + github actions

.github/workflows/build-image.yaml

@@ -0,0 +1,31 @@
+name: build-image
+run-name: Build the shimboot disk image for all boards
+on: [push]
+
+jobs:
+  build:
+    strategy: 
+      matrix:
+        board: [dedede, octopus]
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: download repo
+        uses: actions/checkout@v4
+      
+      - name: run build
+        run: sudo DEBUG=1 ./build_complete.sh ${{ matrix.board }} compress_img=1 quiet_download=1
+
+      - name: upload img
+        uses: actions/upload-artifact@v4
+        with:
+          name: shimboot_${{ matrix.board }}
+          path: data/shimboot_${{ matrix.board }}.zip
+          compression-level: 0
+  
+      - name: create release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          draft: true
+          files: data/shimboot_${{ matrix.board }}.zip

build.sh

@@ -14,15 +14,6 @@ print_help() {
   echo "Usage: ./build.sh output_path shim_path rootfs_dir"
 }
 
-check_deps() {
-  local needed_commands="cpio binwalk pcregrep realpath cgpt mkfs.ext4 mkfs.ext2 fdisk rsync"
-  for command in $needed_commands; do
-    if ! command -v $command &> /dev/null; then
-      echo $command
-    fi
-  done
-}
-
 if [ "$EUID" -ne 0 ]; then
   echo "this needs to be run as root."
   exit 1
@@ -33,17 +24,12 @@ if [ -z "$3" ]; then
   exit 1
 fi
 
-missing_commands=$(check_deps)
+. ./common.sh
-if [ "${missing_commands}" ]; then
+assert_deps "cpio binwalk pcregrep realpath cgpt mkfs.ext4 mkfs.ext2 fdisk rsync"
-  echo "You are missing dependencies needed for this script."
-  echo "Commands needed:"
-  echo "${missing_commands}"
-  exit 1
-fi
 
-output_path=$(realpath "${1}")
+output_path=$(realpath -m "${1}")
-shim_path=$(realpath "${2}")
+shim_path=$(realpath -m "${2}")
-rootfs_dir=$(realpath "${3}")
+rootfs_dir=$(realpath -m "${3}")
 
 echo "created loop device for shim"
 shim_loop=$(create_loop "${shim_path}")

build_complete.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+set -e
+if [ "$DEBUG" ]; then
+  set -x
+  export DEBUG=1
+fi
+
+. ./common.sh
+
+if [ "$EUID" -ne 0 ]; then
+  echo "This script must be run as root."
+  exit 1
+fi
+
+if [ -z "$1" ]; then
+  echo "Usage: ./build_complete.sh board_name"
+  echo "Valid named arguments (specify with 'key=value'):"
+  echo "  compress_img   - Compress the final disk image into a zip file. Set this to any value to enable this option."
+  echo "  rootfs_dir     - Use a different rootfs for the build. The directory you select will be copied before any patches are applied."
+  echo "  quiet_download - Don't use progress bars on downloads."
+  exit 1
+fi
+
+parse_args "$@"
+needed_deps="wget python3 unzip zip git debootstrap cpio binwalk pcregrep cgpt mkfs.ext4 mkfs.ext2 fdisk rsync"
+if [ "$(check_deps "$needed_deps")" ]; then
+  #install deps automatically on debian and ubuntu
+  if [ -f "/etc/debian_version" ]; then
+    echo "attempting to install build deps"
+    apt-get install wget python3-all unzip zip debootstrap cpio binwalk pcregrep cgpt rsync pv -y
+  fi
+  assert_deps "$needed_deps"
+fi
+
+cleanup_path=""
+sigint_handler() {
+  if [ $cleanup_path ]; then
+    rm -rf $cleanup_path
+  fi
+  exit 1
+}
+trap sigint_handler SIGINT
+
+base_dir="$(realpath -m  $(dirname "$0"))"
+board="$1"
+shim_url="https://dl.osu.bio/api/raw/?path=/SH1mmer/$board.zip"
+boards_url="https://chromiumdash.appspot.com/cros/fetch_serving_builds?deviceCategory=ChromeOS"
+
+echo "downloading list of recovery images"
+reco_url="$(wget -qO- --show-progress $boards_url | python3 -c '
+import json, sys
+
+all_builds = json.load(sys.stdin)
+board = all_builds["builds"][sys.argv[1]]
+if "models" in board:
+  board = next(iter(board["models"].values()))
+
+reco_url = list(board["pushRecoveries"].values())[-1]
+print(reco_url)
+' $board)"
+echo "found url: $reco_url"
+
+shim_bin="$base_dir/data/shim_$board.bin"
+shim_zip="$base_dir/data/shim_$board.zip"
+reco_bin="$base_dir/data/reco_$board.bin"
+reco_zip="$base_dir/data/reco_$board.zip"
+mkdir -p "$base_dir/data"
+
+download_and_unzip() {
+  local url="$1"
+  local zip_path="$2"
+  local bin_path="$3"
+  if [ ! -f "$bin_path" ]; then
+    if [ ! "${args['quiet_download']}" ]; then
+      wget -q --show-progress $url -O $zip_path -c
+    else
+      wget -q $url -O $zip_path -c
+    fi
+  fi
+
+  if [ ! -f "$bin_path" ]; then
+    cleanup_path="$bin_path"
+    echo "extracting $zip_path"
+    local total_bytes="$(unzip -lq $zip_path | tail -1 | xargs | cut -d' ' -f1)"
+    if [ ! "${args['quiet_download']}" ]; then
+      unzip -p $zip_path | pv -s $total_bytes > $bin_path
+    else
+      unzip -p $zip_path > $bin_path
+    fi
+    rm -rf $zip_path
+    cleanup_path=""
+  fi
+}
+
+echo "downloading recovery image"
+download_and_unzip $reco_url $reco_zip $reco_bin
+
+echo "downloading shim image"
+download_and_unzip $shim_url $shim_zip $shim_bin
+
+if [ ! "${args['rootfs_dir']}" ]; then
+  rootfs_dir="$(realpath -m data/rootfs_$board)"
+  rm -rf $rootfs_dir
+  mkdir -p $rootfs_dir
+
+  echo "building debian rootfs"
+  ./build_rootfs.sh $rootfs_dir bookworm \
+    hostname=shimboot-$board \
+    root_passwd=root \
+    username=user \
+    user_passwd=user  
+else
+  rootfs_dir="$(realpath -m "${args['rootfs_dir']}")"
+fi
+
+echo "patching debian rootfs"
+./patch_rootfs.sh $shim_bin $reco_bin $rootfs_dir
+
+echo "building final disk image"
+final_image="$base_dir/data/shimboot_$board.bin"
+rm -rf $final_image
+./build.sh $final_image $shim_bin $rootfs_dir
+echo "build complete! the final disk image is located at $final_image"
+
+if [ "${args['compress_img']}" ]; then
+  image_zip="$base_dir/data/shimboot_$board.zip"
+  echo "compressing disk image into a zip file"
+  zip -j $image_zip $final_image
+  echo "finished compressing the disk file"
+  echo "the finished zip file can be found at $image_zip" 
+fi

build_rootfs.sh

@@ -7,55 +7,58 @@ if [ "$DEBUG" ]; then
   set -x
 fi
 
-print_help() {
+. ./common.sh
-  echo "Usage: ./build_rootfs.sh rootfs_path release_name [custom_packages]"
-}
-
-check_deps() {
-  local needed_commands="realpath debootstrap"
-  for command in $needed_commands; do
-    if ! command -v $command &> /dev/null; then
-      echo $command
-    fi
-  done
-}
 
 if [ "$EUID" -ne 0 ]; then
-  echo "this needs to be run as root."
+  echo "This script must be run as root."
   exit 1
 fi
 
 if [ -z "$2" ]; then
-  print_help
+  echo "Usage: ./build_rootfs.sh rootfs_path release_name"
+  echo "Valid named arguments (specify with 'key=value'):"
+  echo "  custom_packages - The packages that will be installed in place of task-xfce-desktop."
+  echo "  hostname        - The hostname for the new rootfs."
+  echo "  root_passwd     - The root password."
+  echo "  username        - The unprivileged user name for the new rootfs."
+  echo "  user_passwd     - The password for the unprivileged user."
+  echo "If you do not specify the hostname and credentials, you will be prompted for them later."
   exit 1
 fi
 
-missing_commands=$(check_deps)
+assert_deps "realpath debootstrap"
-if [ "${missing_commands}" ]; then
+parse_args "$@"
-  echo "You are missing dependencies needed for this script."
-  echo "Commands needed:"
-  echo "${missing_commands}"
-  exit 1
-fi
 
-rootfs_dir=$(realpath "${1}")
+rootfs_dir=$(realpath -m "${1}")
 release_name="${2}"
-packages="${3-'task-xfce-desktop'}"
+packages="${args['custom_packages']-'task-xfce-desktop'}"
+chroot_mounts="proc sys dev run"
+
+mkdir -p $rootfs_dir
+
+unmount_all() {
+  for mountpoint in $chroot_mounts; do
+    umount -l "$rootfs_dir/$mountpoint"
+  done
+}
 
 debootstrap --arch amd64 $release_name $rootfs_dir http://deb.debian.org/debian/
 cp -ar rootfs/* $rootfs_dir
 cp /etc/resolv.conf $rootfs_dir/etc/resolv.conf
 
-chroot_mounts="proc sys dev run"
+trap unmount_all EXIT
 for mountpoint in $chroot_mounts; do
   mount --make-rslave --rbind "/${mountpoint}" "${rootfs_dir}/$mountpoint"
 done
 
-chroot_command="/opt/setup_rootfs.sh '$DEBUG' '$release_name' '$packages'"
+hostname="${args['hostname']}"
-chroot $rootfs_dir /bin/bash -c "${chroot_command}"
+root_passwd="${args['root_passwd']}"
+username="${args['username']}"
+user_passwd="${args['user_passwd']}"
 
-for mountpoint in $chroot_mounts; do
+chroot_command="/opt/setup_rootfs.sh '$DEBUG' '$release_name' '$packages' '$hostname' '$root_passwd' '$username' '$user_passwd'"
-  umount -l "${rootfs_dir}/$mountpoint"
+chroot $rootfs_dir /bin/bash -c "${chroot_command}"
-done
+trap - EXIT
+unmount_all
 
 echo "rootfs has been created"

build_squashfs.sh

@@ -47,9 +47,9 @@ compile_unionfs() {
   cd $original_dir
 }
 
-rootfs_dir=$(realpath $1)
+rootfs_dir=$(realpath -m $1)
-old_dir=$(realpath $2)
+old_dir=$(realpath -m $2)
-shim_path=$(realpath $3)
+shim_path=$(realpath -m $3)
 
 shim_rootfs="/tmp/shim_rootfs"
 root_squashfs="$rootfs_dir/root.squashfs"

common.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+check_deps() {
+  local needed_commands="$1"
+  for command in $needed_commands; do
+    if ! command -v $command &> /dev/null; then
+      echo $command
+    fi
+  done
+}
+
+assert_deps() {
+  local needed_commands="$1"
+  local missing_commands=$(check_deps "$needed_commands")
+  if [ "${missing_commands}" ]; then
+    echo "You are missing dependencies needed for this script."
+    echo "Commands needed:"
+    echo "${missing_commands}"
+    exit 1
+  fi
+}
+
+parse_args() {
+  declare -g -A args
+  for argument in "$@"; do
+    local key=$(echo $argument | cut -f1 -d=)
+    local key_length=${#key}
+    local value="${argument:$key_length+1}"
+    args["$key"]="$value"
+  done
+}

image_utils.sh

@@ -17,7 +17,7 @@ make_bootable() {
 }
 
 partition_disk() {
-  local image_path=$(realpath "${1}")
+  local image_path=$(realpath -m "${1}")
   local bootloader_size=${2}
 
   #create partition table with fdisk
@@ -68,12 +68,13 @@ safe_mount() {
   umount $2 2> /dev/null || /bin/true
   rm -rf $2
   mkdir -p $2
-  mount $1 $2
+  #try to mount multiple times
+  mount $1 $2 || mount $1 $2 || mount $1 $2 || mount $1 $2 
 }
 
 create_partitions() {
-  local image_loop=$(realpath "${1}")
+  local image_loop=$(realpath -m "${1}")
-  local kernel_path=$(realpath "${2}")
+  local kernel_path=$(realpath -m "${2}")
 
   #create stateful
   mkfs.ext4 "${image_loop}p1"
@@ -87,9 +88,9 @@ create_partitions() {
 }
 
 populate_partitions() {
-  local image_loop=$(realpath "${1}")
+  local image_loop=$(realpath -m "${1}")
-  local bootloader_dir=$(realpath "${2}")
+  local bootloader_dir=$(realpath -m "${2}")
-  local rootfs_dir=$(realpath "${3}")
+  local rootfs_dir=$(realpath -m "${3}")
 
   #mount and write empty file to stateful
   local stateful_mount=/tmp/shim_stateful
@@ -113,7 +114,7 @@ populate_partitions() {
 }
 
 create_image() {
-  local image_path=$(realpath "${1}")
+  local image_path=$(realpath -m "${1}")
   local bootloader_size=${2}
   local rootfs_size=${3}
   
@@ -126,7 +127,7 @@ create_image() {
 }
 
 patch_initramfs() {
-  local initramfs_path=$(realpath $1)
+  local initramfs_path=$(realpath -m $1)
 
   rm "${initramfs_path}/init" -f
   cp -r bootloader/* "${initramfs_path}/"

patch_rootfs.sh

@@ -24,9 +24,9 @@ if [ -z "$3" ]; then
 fi
 
 copy_modules() {
-  local shim_rootfs=$(realpath $1)
+  local shim_rootfs=$(realpath -m $1)
-  local reco_rootfs=$(realpath $2)
+  local reco_rootfs=$(realpath -m $2)
-  local target_rootfs=$(realpath $3)
+  local target_rootfs=$(realpath -m $3)
 
   rm -rf "${target_rootfs}/lib/modules"
   cp -r "${shim_rootfs}/lib/modules" "${target_rootfs}/lib/modules"
@@ -43,7 +43,7 @@ copy_modules() {
 
 copy_firmware() {
   local firmware_path="/tmp/chromium-firmware"
-  local target_rootfs=$(realpath $1)
+  local target_rootfs=$(realpath -m $1)
 
   if [ ! -e "$firmware_path" ]; then
     download_firmware $firmware_path
@@ -54,14 +54,14 @@ copy_firmware() {
 
 download_firmware() {
   local firmware_url="https://chromium.googlesource.com/chromiumos/third_party/linux-firmware"
-  local firmware_path=$(realpath $1)
+  local firmware_path=$(realpath -m $1)
 
   git clone --branch master --depth=1 "${firmware_url}" $firmware_path
 }
 
-shim_path=$(realpath $1)
+shim_path=$(realpath -m $1)
-reco_path=$(realpath $2)
+reco_path=$(realpath -m $2)
-target_rootfs=$(realpath $3)
+target_rootfs=$(realpath -m $3)
 shim_rootfs="/tmp/shim_rootfs"
 reco_rootfs="/tmp/reco_rootfs"
 

rootfs/opt/setup_rootfs.sh

@@ -3,19 +3,26 @@
 #setup the debian rootfs
 #this is meant to be run within the chroot created by debootstrap
 
-DEBUG="$1"
-release_name="$2"
-packages="$3"
-
 set -e
 if [ "$DEBUG" ]; then
   set -x
 fi
 
+DEBUG="$1"
+release_name="$2"
+packages="$3"
+
+hostname="$4"
+root_passwd="$5"
+username="$6"
+user_passwd="$7"
+
 custom_repo="https://shimboot.ading.dev/debian"
 custom_repo_domain="shimboot.ading.dev"
 sources_entry="deb [trusted=yes arch=amd64] ${custom_repo} ${release_name} main"
 
+export DEBIAN_FRONTEND=noninteractive
+
 #add shimboot repos
 echo -e "${sources_entry}\n$(cat /etc/apt/sources.list)" > /etc/apt/sources.list
 tee -a /etc/apt/preferences << END
@@ -43,7 +50,9 @@ PERCENT=50
 END
 
 #set up hostname and username
-read -p "Enter the hostname for the system: " hostname
+if [ ! "$hostname" ]; then
+  read -p "Enter the hostname for the system: " hostname
+fi
 echo "${hostname}" > /etc/hostname
 tee -a /etc/hosts << END
 127.0.0.1 localhost
@@ -56,16 +65,27 @@ ff02::2 ip6-allrouters
 END
 
 echo "Enter a root password:"
-while ! passwd root; do
+if [ ! "$root_passwd" ]; then
-  echo "Failed to set password, please try again."
+  while ! passwd root; do
-done
+    echo "Failed to set password, please try again."
+  done
+else
+  yes "$root_passwd" | passwd root
+fi
 
-read -p "Enter the username for the user account: " username
+if [ ! $username ]; then
+  read -p "Enter the username for the user account: " username
+fi
 useradd -m -s /bin/bash -G sudo $username
-echo "Enter the password for ${username}:"
+
-while ! passwd $username; do
+if [ ! "$user_passwd" ]; then
-  echo "Failed to set password, please try again."
+  echo "Enter the password for ${username}:"
-done
+  while ! passwd $username; do
+    echo "Failed to set password, please try again."
+  done
+else
+  yes "$user_passwd" | passwd $username
+fi
 
 #clean apt caches
 apt-get clean