From ea3a3b10a0970a598da12fd6907961d47457f385 Mon Sep 17 00:00:00 2001
From: Ruoqing He <heruoqing@iscas.ac.cn>
Date: Sun, 29 Sep 2024 02:32:11 +0000
Subject: [PATCH] Use raw image to boot VM

Signed-off-by: Ruoqing He <heruoqing@iscas.ac.cn>
---
 .github/workflows/docker-publish.yml          | 18 +++++++++++++++
 Dockerfile.riscv64                            | 16 +++----------
 Dockerfile.riscv64_rootfs                     | 22 ++++++++++++++++++
 riscv64/build_qemu_system_riscv64.sh          |  2 +-
 riscv64/convert.sh                            | 23 +++++++++++++++++++
 .../{build_finalize.sh => rootfs_finalize.sh} |  9 ++++++++
 riscv64/start_in_qemu.sh                      | 15 ++++++------
 7 files changed, 84 insertions(+), 21 deletions(-)
 create mode 100644 Dockerfile.riscv64_rootfs
 create mode 100755 riscv64/convert.sh
 rename riscv64/{build_finalize.sh => rootfs_finalize.sh} (66%)

diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index 95a9b6d..b0a29e9 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -117,6 +117,24 @@ jobs:
           echo "VERSION=${NEXT_VERSION}" >> $GITHUB_ENV
           echo "Next version to be published is: ${NEXT_VERSION}"
 
+      - name: Build RISC-V rootfs
+        id: build-riscv-rootfs
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.riscv64_rootfs
+          push: false
+          load: true
+          platforms: linux/amd64
+          tags: rootfs
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Convert rootfs to raw image
+        run: |
+          mkdir -p ./extracted
+          docker run --privileged --volume ./riscv64/convert.sh:/convert.sh --volume ./extracted:/to_extract --entrypoint /convert.sh rootfs
+
       - name: Build and push Docker image for RISC-V
         id: build-and-push-riscv
         uses: docker/build-push-action@v6
diff --git a/Dockerfile.riscv64 b/Dockerfile.riscv64
index 7bdc624..a62e153 100644
--- a/Dockerfile.riscv64
+++ b/Dockerfile.riscv64
@@ -1,4 +1,4 @@
-# Compile QEMU 9.0.2
+# Compile QEMU 9.1.0
 # ---------------------------------------------------------
 FROM ubuntu:22.04 AS qemu_builder
 
@@ -19,15 +19,6 @@ FROM ubuntu:22.04 AS opensbi_builder
 COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh
 RUN /opt/src/scripts/build.sh
 
-# Build rootfs with sshd and Rust related packages ready
-# ---------------------------------------------------------
-FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
-
-ARG RUST_TOOLCHAIN="1.75.0"
-ENV PATH="$PATH:/root/.cargo/bin"
-COPY build_container.sh /opt/src/scripts/build.sh
-RUN /opt/src/scripts/build.sh
-
 # Finalize
 # ---------------------------------------------------------
 FROM ubuntu:22.04 AS final
@@ -37,14 +28,13 @@ ARG QEMU_DIR=/opt/qemu
 ARG KERNEL_DIR=/opt/kernel
 ARG OPENSBI_DIR=/opt/opensbi
 ARG ROOTFS_DIR=/opt/rootfs
+ARG EXTRACT_DIR=extracted
 
 COPY --from=qemu_builder $OUTPUT $QEMU_DIR
 COPY --from=kernel_builder $OUTPUT $KERNEL_DIR
 COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR
-COPY --from=rootfs_builder / $ROOTFS_DIR
 
-COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh
-RUN /opt/src/scripts/finalize.sh
+COPY $EXTRACT_DIR/* $ROOTFS_DIR
 
 ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \
     OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \
diff --git a/Dockerfile.riscv64_rootfs b/Dockerfile.riscv64_rootfs
new file mode 100644
index 0000000..4d5ea60
--- /dev/null
+++ b/Dockerfile.riscv64_rootfs
@@ -0,0 +1,22 @@
+# Build rootfs with sshd and Rust related packages ready
+# ---------------------------------------------------------
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder
+
+ARG RUST_TOOLCHAIN="1.75.0"
+ENV PATH="$PATH:/root/.cargo/bin"
+COPY build_container.sh /opt/src/scripts/build.sh
+RUN /opt/src/scripts/build.sh
+
+# Finalize
+# ---------------------------------------------------------
+FROM ubuntu:22.04 AS final
+
+ARG OUTPUT=/output
+ARG ROOTFS_DIR=/opt/rootfs
+
+COPY --from=rootfs_builder / $ROOTFS_DIR
+
+COPY riscv64/rootfs_finalize.sh /opt/src/scripts/finalize.sh
+RUN /opt/src/scripts/finalize.sh
+
+ENV ROOTFS_DIR=$ROOTFS_DIR
diff --git a/riscv64/build_qemu_system_riscv64.sh b/riscv64/build_qemu_system_riscv64.sh
index d42a5a3..154692a 100755
--- a/riscv64/build_qemu_system_riscv64.sh
+++ b/riscv64/build_qemu_system_riscv64.sh
@@ -3,7 +3,7 @@ set -ex
 
 apt-get update
 
-QEMU_TAG=v9.0.2
+QEMU_TAG=v9.1.0
 OUTPUT=/output
 mkdir $OUTPUT
 
diff --git a/riscv64/convert.sh b/riscv64/convert.sh
new file mode 100755
index 0000000..ac946b3
--- /dev/null
+++ b/riscv64/convert.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -ex
+
+apt-get update
+
+DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    qemu-utils
+
+TMP_MOUNT_DIR=tmp_mount_dir
+ROOTFS_DIR=/opt/rootfs
+EXTRACT_DIR=/to_extract
+
+# Move rootfs into image with raw format
+qemu-img create $EXTRACT_DIR/rootfs.img 5G && mkfs.ext4 $EXTRACT_DIR/rootfs.img
+mkdir $TMP_MOUNT_DIR
+# This operation need `privileged` docker container
+mount -o loop $EXTRACT_DIR/rootfs.img $TMP_MOUNT_DIR
+mv $ROOTFS_DIR/* $TMP_MOUNT_DIR
+umount $TMP_MOUNT_DIR
+rmdir $TMP_MOUNT_DIR
+
+mv /root/.ssh $EXTRACT_DIR
+chmod 777 $EXTRACT_DIR/.ssh
diff --git a/riscv64/build_finalize.sh b/riscv64/rootfs_finalize.sh
similarity index 66%
rename from riscv64/build_finalize.sh
rename to riscv64/rootfs_finalize.sh
index ba2d9ce..c72641d 100755
--- a/riscv64/build_finalize.sh
+++ b/riscv64/rootfs_finalize.sh
@@ -7,6 +7,15 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
     openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \
     libpixman-1-dev netcat
 
+# Create `/root/workdir` as mount point of `9p` fs
+mkdir $ROOTFS_DIR/root/workdir
+
+# Modify fstab to mount `workdir` during boot
+echo "workdir	/root/workdir	9p	trans=virtio,version=9p2000.L	0	0" > $ROOTFS_DIR/etc/fstab
+
+# Export rust components to PATH
+echo $'\nexport PATH="$PATH:/root/.cargo/bin"' >> $ROOTFS_DIR/root/.bashrc
+
 # Setup container ssh config
 yes "" | ssh-keygen -P ""
 cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys
diff --git a/riscv64/start_in_qemu.sh b/riscv64/start_in_qemu.sh
index 7f4ac3c..171a75a 100755
--- a/riscv64/start_in_qemu.sh
+++ b/riscv64/start_in_qemu.sh
@@ -20,12 +20,13 @@ $QEMU_DIR/bin/qemu-system-riscv64 \
     -smp $CORES -m $MEM \
     -bios $OPENSBI_DIR/fw_jump.elf \
     -kernel $KERNEL_DIR/Image \
-    -device virtio-net-device,netdev=usernet -netdev user,id=usernet,hostfwd=tcp::2222-:22 \
-    -virtfs local,path=$ROOTFS_DIR,mount_tag=rootfs,security_model=none,id=rootfs \
-    -append "root=rootfs rw rootfstype=9p rootflags=trans=virtio,cache=mmap,msize=512000 console=ttyS0 earlycon=sbi nokaslr rdinit=/sbin/init" 2>&1 &
-
-# Copy WORKDIR to ROOTFS_DIR
-cp -a $WORKDIR $ROOTFS_DIR/root
+    -device virtio-net-device,netdev=usernet \
+    -netdev user,id=usernet,hostfwd=tcp::2222-:22 \
+    -drive file=$ROOTFS_DIR/rootfs.img,format=raw,id=rootfs \
+    -device virtio-blk-device,drive=rootfs \
+    -fsdev local,security_model=passthrough,id=fsdev0,path=$WORKDIR \
+    -device virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=workdir \
+    -append "root=/dev/vda rw console=ttyS0 earlycon=sbi nokaslr rdinit=/sbin/init"
 
 HOST=riscv-qemu
 
@@ -36,4 +37,4 @@ done
 
 # Issue command
 COMMAND=$@
-ssh $HOST "export PATH=\"\$PATH:/root/.cargo/bin\" && cd workdir && $COMMAND"
+ssh $HOST "cd workdir && $COMMAND"