Merge pull request #3 from ading2210/squashfs-testing

Add squashfs boot support

.gitignore

@@ -1,2 +1,2 @@
 /data
-/test.bin
+/*.bin

README.md

@@ -33,9 +33,10 @@ Driver support depends on the device you are using shimboot on. This list is for
 - Bluetooth
 - Zram
 - Wifi
+- Booting a squashfs
 
 ### What Doesn't Work:
-- Audio
+- Audio (due to a firmware bug)
 - Suspend (disabled by the kernel)
 - Swap (disabled by the kernel)
 
@@ -91,11 +92,14 @@ Driver support depends on the device you are using shimboot on. This list is for
 Using any Linux distro is possible, provided that you apply the [proper patches](https://github.com/ading2210/chromeos-systemd) to systemd and recompile it. Most distros have some sort of bootstrapping tool that allows you to install it to a directory on your host PC. Then, you can just pass that rootfs dir into `build.sh`.
 
 #### How can I install a desktop environment other than XFCE?
-Simply edit `rootfs/opt/setup_rootfs.sh`, and change the line after the `#install desktop` comment. By default, this is set to install XFCE using the `task-xfce-desktop` package, but you can change this to install whatever you want.
+You can pass another argument to the `build_rootfs.sh` script, like this: `sudo ./build_rootfs.sh data/rootfs bookworm "task-lxde-desktop"`. The third argument is a list of packages that will be installed in the place of XFCE. 
 
 #### Will this prevent me from using Chrome OS normally?
 Shimboot does not touch the internal storage at all, so you will be able to use Chrome OS as if nothing happened. However, if you are on an enterprise enrolled device, booting Chrome OS again will force a powerwash due to the attempted switch into developer mode.
 
+#### Can I unplug the USB drive while using Debian?
+By default, this is not possible. However, you can simply copy your Debian rootfs onto your internal storage by first using `fdisk` to repartition it, using `dd` to copy the partition, and `resize2fs` to have it take up the entire drive. In the future, loading the OS to RAM may be supported, but this isn't a priority at the moment.
+
 ## Copyright:
 Shimboot is licensed under the [GNU GPL v3](https://www.gnu.org/licenses/gpl-3.0.txt). Unless otherwise indicated, all code has been written by me, [ading2210](https://github.com/ading2210).
 

build.sh

@@ -7,8 +7,8 @@ if [ "$DEBUG" ]; then
   set -x
 fi
 
-. ./patch_initramfs.sh
-. ./build_image.sh
+. ./image_utils.sh
+. ./shim_utils.sh
 
 print_help() {
   echo "Usage: ./build.sh output_path shim_path rootfs_dir"
@@ -55,34 +55,10 @@ mkdir $kernel_dir -p
 dd if=$kernel_loop of=$kernel_dir/kernel.bin bs=1M status=none
 
 echo "extracting data from kernel"
-previous_dir=$(pwd)
-cd $kernel_dir
-if [ -e "${kernel_dir}/binwalk.out" ]; then
-  #don't run binwalk again if we don't need to
-  binwalk_out=$(cat $kernel_dir/binwalk.out)
-else
-  binwalk_out=$(binwalk --extract kernel.bin --run-as=root)
-  echo $binwalk_out > $kernel_dir/binwalk.out
-fi
-#i can't be bothered to learn how to use sed
-extracted_file=$(echo $binwalk_out | pcregrep -o1 "\d+\s+0x([0-9A-F]+)\s+gzip compressed data")
-
-echo "extracting initramfs archive from kernel (this may take a while)"
-cd _kernel.bin.extracted/
-if [ ! -e "_${extracted_file}.extracted/" ]; then
-  binwalk --extract $extracted_file --run-as=root > /dev/null
-fi
-cd "_${extracted_file}.extracted/"
-cpio_file=$(file ./* | pcregrep -o1 "([0-9A-F]+):\s+ASCII cpio archive")
-
-echo "extracting initramfs cpio archive"
 initramfs_dir=/tmp/shim_initramfs
 rm -rf $initramfs_dir
-cat $cpio_file | cpio -D $initramfs_dir -imd --quiet
-echo "shim initramfs extracted to ${initramfs_dir}"
-
-#leave /tmp
-cd $previous_dir
+extract_initramfs $kernel_dir/kernel.bin $kernel_dir $initramfs_dir
+losetup -d $shim_loop
 
 echo "patching initramfs"
 patch_initramfs $initramfs_dir
@@ -100,15 +76,9 @@ image_loop=$(create_loop ${output_path})
 echo "creating partitions on the disk image"
 create_partitions $image_loop "${kernel_dir}/kernel.bin"
 
-echo "mounting the original shim rootfs"
-shim_rootfs="/tmp/shim_rootfs"
-make_mountable "${shim_loop}p3"
-safe_mount "${shim_loop}p3" $shim_rootfs
-
 echo "copying data into the image"
 populate_partitions $image_loop $initramfs_dir $rootfs_dir
 
 echo "cleaning up loop devices"
-losetup -d $shim_loop
 losetup -d $image_loop
 echo "done"

build_rootfs.sh

@@ -8,7 +8,7 @@ if [ "$DEBUG" ]; then
 fi
 
 print_help() {
-  echo "Usage: ./build_rootfs.sh rootfs_path release_name"
+  echo "Usage: ./build_rootfs.sh rootfs_path release_name [custom_packages]"
 }
 
 check_deps() {
@@ -40,16 +40,18 @@ fi
 
 rootfs_dir=$(realpath "${1}")
 release_name="${2}"
+packages="${3-'task-xfce-desktop'}"
 
 debootstrap --arch amd64 $release_name $rootfs_dir http://deb.debian.org/debian/
 cp -ar rootfs/* $rootfs_dir
+cp /etc/resolv.conf $rootfs_dir/etc/resolv.conf
 
 chroot_mounts="proc sys dev run"
 for mountpoint in $chroot_mounts; do
   mount --make-rslave --rbind "/${mountpoint}" "${rootfs_dir}/$mountpoint"
 done
 
-chroot_command="DEBUG=${DEBUG} release_name=${release_name} /opt/setup_rootfs.sh"
+chroot_command="/opt/setup_rootfs.sh '$DEBUG' '$release_name' '$packages'"
 chroot $rootfs_dir /bin/bash -c "${chroot_command}"
 
 for mountpoint in $chroot_mounts; do

build_squashfs.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+#build a rootfs that uses a squashfs + unionfs
+#consists of a minimal busybox system containing:
+# - FUSE kernel modules from the shim
+# - unionfs-fuse statically compiled 
+# - the main squashfs, compressed with gzip
+#this script is currently incomplete
+
+set -e
+if [ "$DEBUG" ]; then
+  set -x
+fi
+
+. ./image_utils.sh
+. ./shim_utils.sh
+
+print_help() {
+  echo "Usage: ./build_squashfs.sh rootfs_dir uncompressed_rootfs_dir path_to_shim"
+}
+
+if [ "$EUID" -ne 0 ]; then
+  echo "this needs to be run as root."
+  exit 1
+fi
+
+if [ -z "$3" ]; then
+  print_help
+  exit 1
+fi
+
+compile_unionfs() {
+  local out_path="$1"
+  local working_path="$2"
+
+  local repo_url="https://github.com/rpodgorny/unionfs-fuse"
+  local original_dir="$(pwd)"
+  local core_count="$(nproc --all)"
+  
+  rm -rf $working_path
+  git clone $repo_url -b master --depth=1 $working_path
+  cd $working_path
+
+  env LDFLAGS="-static" make -j$core_count
+  local binary_path="$working_path/src/unionfs"
+  cp $binary_path $out_path
+  cd $original_dir
+}
+
+rootfs_dir=$(realpath $1)
+old_dir=$(realpath $2)
+shim_path=$(realpath $3)
+
+shim_rootfs="/tmp/shim_rootfs"
+root_squashfs="$rootfs_dir/root.squashfs"
+modules_squashfs="$rootfs_dir/modules.squashfs"
+kernel_dir=/tmp/shim_kernel
+unionfs_dir="/tmp/unionfs-fuse"
+
+echo "compiling unionfs-fuse"
+compile_unionfs $unionfs_dir/unionfs $unionfs_dir
+
+echo "creating loop device for shim"
+shim_loop=$(create_loop "${shim_path}")
+kernel_loop="${shim_loop}p2" #KERN-A should always be p2
+
+echo "copying shim kernel"
+rm -rf $kernel_dir
+mkdir $kernel_dir -p
+dd if=$kernel_loop of=$kernel_dir/kernel.bin bs=1M status=progress
+
+echo "extracting initramfs from kernel (this may take a while)"
+extract_initramfs $kernel_dir/kernel.bin $kernel_dir $rootfs_dir
+rm -rf $rootfs_dir/init
+
+echo "removeing shim loop device"
+losetup -d $shim_loop
+
+echo "compressing old rootfs"
+mksquashfs $old_dir $root_squashfs -noappend -comp gzip
+
+echo "patching new rootfs"
+mv $unionfs_dir/unionfs $rootfs_dir/bin/unionfs
+cp -ar squashfs/* $rootfs_dir/
+chmod +x $rootfs_dir/bin/*
+
+echo "done"

image_utils.sh

@@ -1,10 +1,5 @@
 #!/bin/bash
 
-set -e
-if [ "$DEBUG" ]; then
-  set -x
-fi
-
 create_loop() {
   local loop_device=$(losetup -f)
   losetup -P $loop_device "${1}"
@@ -128,4 +123,13 @@ create_image() {
   fallocate -l "${total_size}M" "${image_path}"
 
   partition_disk $image_path $bootloader_size
+}
+
+patch_initramfs() {
+  local initramfs_path=$(realpath $1)
+
+  rm "${initramfs_path}/init" -f
+  cp -r bootloader/* "${initramfs_path}/"
+
+  find ${initramfs_path}/bin -name "*" -exec chmod +x {} \;
 }

patch_initramfs.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-#patch the shim initramfs to add the bootloader
-
-set -e
-if [ "$DEBUG" ]; then
-  set -x
-fi
-
-patch_initramfs() {
-  local initramfs_path=$(realpath $1)
-
-  rm "${initramfs_path}/init" -f
-  cp -r bootloader/* "${initramfs_path}/"
-
-  find ${initramfs_path}/bin -name "*" -exec chmod +x {} \;
-}

patch_rootfs.sh

@@ -7,7 +7,7 @@ if [ "$DEBUG" ]; then
   set -x
 fi
 
-. ./build_image.sh
+. ./image_utils.sh
 
 print_help() {
   echo "Usage: ./patch_rootfs.sh shim_path reco_path rootfs_dir"

rootfs/opt/setup_rootfs.sh

@@ -3,6 +3,10 @@
 #setup the debian rootfs
 #this is meant to be run within the chroot created by debootstrap
 
+DEBUG="$1"
+release_name="$2"
+packages="$3"
+
 set -e
 if [ "$DEBUG" ]; then
   set -x
@@ -23,13 +27,14 @@ END
 #install the patched systemd
 apt-get install -y ca-certificates
 apt-get update
-apt-get upgrade -y --allow-downgrades
+installed_systemd="$(dpkg-query -W -f='${binary:Package}\n' | grep "systemd")"
+apt-get install --reinstall $installed_systemd
 
 #enable shimboot services
 systemctl enable kill-frecon.service
 
 #install desktop
-apt-get install -y task-xfce-desktop cloud-utils zram-tools --allow-downgrades
+apt-get install -y $packages cloud-utils zram-tools
 
 #set up zram
 tee -a /etc/default/zramswap << END

shim_utils.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#utilties for reading shim disk images
+
+extract_initramfs() {
+  local kernel_bin="$1"
+  local working_dir="$2"
+  local output_dir="$3"
+
+  #first stage
+  local kernel_file="$(basename $kernel_bin)"
+  local binwalk_out=$(binwalk --extract $kernel_bin --directory=$working_dir --run-as=root)
+  local stage1_file=$(echo $binwalk_out | pcregrep -o1 "\d+\s+0x([0-9A-F]+)\s+gzip compressed data")
+  local stage1_dir="$working_dir/_$kernel_file.extracted"
+  local stage1_path="$stage1_dir/$stage1_file"
+  
+  #second stage
+  binwalk --extract $stage1_path --directory=$stage1_dir --run-as=root > /dev/null
+  local stage2_dir="$stage1_dir/_$stage1_file.extracted/"
+  local cpio_file=$(file $stage2_dir/* | pcregrep -o1 "([0-9A-F]+):\s+ASCII cpio archive")
+  local cpio_path="$stage2_dir/$cpio_file"
+
+  rm -rf $output_dir
+  cat $cpio_path | cpio -D $output_dir -imd --quiet
+}

squashfs/bin/bootstrap.sh

@@ -0,0 +1,68 @@
+#!/bin/busybox sh
+
+#mount the squashfs + unionfs and boot into it
+
+move_mounts() {
+  local base_mounts="/sys /proc /dev"
+  local newroot_mnt="$1"
+  for mnt in $base_mounts; do
+    mkdir -p "$newroot_mnt$mnt"
+    mount -n -o move "$mnt" "$newroot_mnt$mnt"
+  done
+}
+
+boot_dir() {
+  local target="$1"
+
+  echo "moving mounts to newroot"
+  move_mounts $target
+
+  echo "switching root"
+  mkdir -p $target/oldroot
+  pivot_root $target $target/oldroot
+
+  /bin/bash -c "mount -o bind /oldroot/lib/modules /lib/modules"
+  exec /sbin/init < "$TTY1" >> "$TTY1" 2>&1
+}
+
+mount_squashfs() {
+  mkdir -p /lib/modules /mnt/root_squashfs
+  mount /root.squashfs /mnt/root_squashfs
+  mount -o bind /mnt/root_squashfs/lib/modules /lib/modules
+}
+
+#based on https://github.com/rpodgorny/unionfs-fuse/blob/master/examples/S01a-unionfs-live-cd.sh
+mount_unionfs() {
+  local chroot_path="/tmp/unionfs"
+  local data_path="/data"
+  local mountpoint="/newroot"
+  local squashfs_path="/mnt/root_squashfs"
+
+  local fuse_options="-o allow_other,suid,dev"
+  local unionfs_options="-o cow,chroot=$chroot_path,max_files=32768"
+
+  mkdir -p $data_path 
+  mkdir -p $mountpoint
+  mkdir -p $chroot_path/root
+  mkdir -p $chroot_path/rw
+  
+  mount -o bind $squashfs_path $chroot_path/root
+  mount -o bind $data_path $chroot_path/rw
+
+  modprobe fuse
+  unionfs $fuse_options $unionfs_options /root=RO:/rw=RW /newroot
+}
+
+main() {
+  echo "mounting squashfs"
+  mount_squashfs
+
+  echo "mounting unionfs"
+  mount_unionfs
+
+  echo "booting unionfs"
+  boot_dir /newroot
+}
+
+main "$@"
+sleep 1d

squashfs/bin/init

@@ -0,0 +1,29 @@
+#!/bin/busybox sh
+
+#original: https://chromium.googlesource.com/chromiumos/platform/initramfs/+/refs/heads/main/factory_shim/init
+
+detect_tty() {
+  if [ -f "/bin/frecon-lite" ]; then
+    export TTY1="/dev/pts/0"
+    export TTY2="/dev/pts/1"
+  else
+    export TTY1="/dev/tty1"
+    export TTY2="/dev/tty2"
+  fi
+}
+
+setup_environment() {
+  # Install additional utility programs.
+  /bin/busybox --install /bin || true
+}
+
+main() {
+  setup_environment
+  detect_tty
+  # In case an error is not handled by bootstrapping, stop here
+  # so that an operator can see installation stop.
+  exec bootstrap.sh < "$TTY1" >> "$TTY1" 2>&1 || sleep 1d
+}
+
+main "$@"
+exit 1