diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b978031..90c0434 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -3,10 +3,20 @@ name: Docker on: push: branches: [ "main" ] - paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh] + paths: + - Dockerfile + - .github/workflows/docker-publish.yml + - build_container.sh + - Dockerfile.riscv64 + - riscv64/* pull_request: branches: [ "main" ] - paths: [Dockerfile, .github/workflows/docker-publish.yml, build_container.sh] + paths: + - Dockerfile + - .github/workflows/docker-publish.yml + - build_container.sh + - Dockerfile.riscv64 + - riscv64/* jobs: build: @@ -69,6 +79,18 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Build and push Docker image for riscv64 + id: build-and-push-riscv64 + uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + with: + context: . + file: Dockerfile.riscv64 + push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64 + tags: ${{ env.VERSION }}-riscv + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker @@ -81,4 +103,6 @@ jobs: COSIGN_EXPERIMENTAL: "true" # This step uses the identity token to provision an ephemeral certificate # against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + run: | + echo "${{ env.VERSION }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + echo "${{ env.VERSION }}-riscv" | xargs -I {} cosign sign {}@${{ steps.build-and-push-riscv.outputs.digest }} diff --git a/Dockerfile.riscv64 b/Dockerfile.riscv64 new file mode 100644 index 0000000..7bdc624 --- /dev/null +++ b/Dockerfile.riscv64 @@ -0,0 +1,55 @@ +# Compile QEMU 9.0.2 +# --------------------------------------------------------- +FROM ubuntu:22.04 AS qemu_builder + +COPY riscv64/build_qemu_system_riscv64.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Compile kernel 6.10 since we need AIA drivers +# --------------------------------------------------------- +FROM ubuntu:22.04 AS kernel_builder + +COPY riscv64/build_kernel.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Compile OpenSBI +# --------------------------------------------------------- +FROM ubuntu:22.04 AS opensbi_builder + +COPY riscv64/build_opensbi.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Build rootfs with sshd and Rust related packages ready +# --------------------------------------------------------- +FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS rootfs_builder + +ARG RUST_TOOLCHAIN="1.75.0" +ENV PATH="$PATH:/root/.cargo/bin" +COPY build_container.sh /opt/src/scripts/build.sh +RUN /opt/src/scripts/build.sh + +# Finalize +# --------------------------------------------------------- +FROM ubuntu:22.04 AS final + +ARG OUTPUT=/output +ARG QEMU_DIR=/opt/qemu +ARG KERNEL_DIR=/opt/kernel +ARG OPENSBI_DIR=/opt/opensbi +ARG ROOTFS_DIR=/opt/rootfs + +COPY --from=qemu_builder $OUTPUT $QEMU_DIR +COPY --from=kernel_builder $OUTPUT $KERNEL_DIR +COPY --from=opensbi_builder $OUTPUT $OPENSBI_DIR +COPY --from=rootfs_builder / $ROOTFS_DIR + +COPY riscv64/build_finalize.sh /opt/src/scripts/finalize.sh +RUN /opt/src/scripts/finalize.sh + +ENV QEMU_DIR=$QEMU_DIR KERNEL_DIR=$KERNEL_DIR \ + OPENSBI_DIR=$OPENSBI_DIR ROOTFS_DIR=$ROOTFS_DIR \ + WORKDIR=/workdir + +# Start qemu-system-riscv64 as a background process +COPY riscv64/start_in_qemu.sh /opt/src/scripts/start.sh +ENTRYPOINT ["/opt/src/scripts/start.sh"] diff --git a/build_container.sh b/build_container.sh index 20249b6..61913d9 100755 --- a/build_container.sh +++ b/build_container.sh @@ -22,15 +22,24 @@ DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ libvirglrenderer-dev libvirglrenderer1 \ debhelper-compat libdbus-1-dev libglib2.0-dev meson ninja-build dbus +# `riscv64` specific +if [ "$ARCH" == "riscv64" ]; then + DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + openssh-server systemd init ifupdown busybox udev isc-dhcp-client +fi + # cleanup apt-get clean && rm -rf /var/lib/apt/lists/* # help musl-gcc find linux headers -pushd /usr/include/$ARCH-linux-musl -ln -s ../$ARCH-linux-gnu/asm asm -ln -s ../linux linux -ln -s ../asm-generic asm-generic -popd +# Skip on `riscv64` for now +if [ "$ARCH" != "riscv64" ]; then + pushd /usr/include/$ARCH-linux-musl + ln -s ../$ARCH-linux-gnu/asm asm + ln -s ../linux linux + ln -s ../asm-generic asm-generic + popd +fi pip3 install --no-cache-dir pytest pexpect boto3 pytest-timeout && apt purge -y python3-pip @@ -52,7 +61,10 @@ rustup component add miri rust-src --toolchain nightly rustup component add llvm-tools-preview # needed for coverage # Install other rust targets. -rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none +# Skip on `riscv64` for now +if [ "$ARCH" != "riscv64" ]; then + rustup target add $ARCH-unknown-linux-musl $ARCH-unknown-none +fi cargo install cargo-llvm-cov @@ -95,3 +107,17 @@ popd # dbus-daemon expects this folder mkdir /run/dbus + +# `riscv64` specific +if [ "$ARCH" == "riscv64" ]; then + # Set passwd for debugging + echo 'root:rustvmm' | chpasswd + # Allow root login + sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config + sed -i 's/#PermitUserEnvironment no/PermitUserEnvironment yes/g' /etc/ssh/sshd_config + # Enable ssh + systemctl enable ssh + mkdir -p /root/.ssh + # Setup network + echo $'auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n' > /etc/network/interfaces +fi diff --git a/docker.sh b/docker.sh index 3f3fa0a..96b0e9a 100755 --- a/docker.sh +++ b/docker.sh @@ -7,11 +7,12 @@ DOCKER_TAG=rustvmm/dev # Get the latest published version. Returns a number. # If latest is v100, returns 100. +# If latest for riscv64 is v100-riscv, returns 100. # This works as long as we have less than 100 tags because we set the page size to 100, # once we have more than that this script needs to be updated. latest(){ curl -L -s 'https://registry.hub.docker.com/v2/repositories/rustvmm/dev/tags?page_size=100'| \ - jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+$" | sort -n | tail -1 + jq '."results"[]["name"]' | sed 's/"//g' | cut -c 2- | grep -E "^[0-9]+" | sort -n | tail -1 } next_version() { diff --git a/riscv64/build_finalize.sh b/riscv64/build_finalize.sh new file mode 100755 index 0000000..5dd8ba3 --- /dev/null +++ b/riscv64/build_finalize.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + openssh-client libslirp-dev libfdt-dev libglib2.0-dev libssl-dev \ + libpixman-1-dev netcat + +# Setup container ssh config +yes "" | ssh-keygen -P "" +cat /root/.ssh/id_rsa.pub > $ROOTFS_DIR/root/.ssh/authorized_keys +cat > /root/.ssh/config << EOF +Host riscv-qemu + HostName localhost + User root + Port 2222 + StrictHostKeyChecking no +EOF + +# Set `nameserver` for `resolv.conf` +echo 'nameserver 8.8.8.8' > $ROOTFS_DIR/etc/resolv.conf diff --git a/riscv64/build_kernel.sh b/riscv64/build_kernel.sh new file mode 100755 index 0000000..412de9d --- /dev/null +++ b/riscv64/build_kernel.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +KERNEL_TAG=v6.10 +OUTPUT=/output +mkdir $OUTPUT + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \ + libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \ + flex bison gcc-riscv64-linux-gnu + +git clone --depth 1 --branch $KERNEL_TAG https://github.com/torvalds/linux.git +pushd linux +# Enable kvm module instead of inserting manually +sed -i "s|^CONFIG_KVM=.*|CONFIG_KVM=y|g" arch/riscv/configs/defconfig +make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- defconfig && \ +make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- -j$(nproc) +mv arch/riscv/boot/Image $OUTPUT +popd diff --git a/riscv64/build_opensbi.sh b/riscv64/build_opensbi.sh new file mode 100755 index 0000000..ac80e17 --- /dev/null +++ b/riscv64/build_opensbi.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +OPENSBI_TAG=v1.3.1 +OUTPUT=/output +mkdir $OUTPUT + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \ + libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \ + flex bison gcc-riscv64-linux-gnu + +git clone --depth 1 --branch $OPENSBI_TAG https://github.com/riscv-software-src/opensbi.git +pushd opensbi +make -j$(nproc) PLATFORM=generic CROSS_COMPILE=riscv64-linux-gnu- +mv build/platform/generic/firmware/fw_jump.elf $OUTPUT +popd diff --git a/riscv64/build_qemu_system_riscv64.sh b/riscv64/build_qemu_system_riscv64.sh new file mode 100755 index 0000000..d42a5a3 --- /dev/null +++ b/riscv64/build_qemu_system_riscv64.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +apt-get update + +QEMU_TAG=v9.0.2 +OUTPUT=/output +mkdir $OUTPUT + +DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ + git python3 python3-pip ninja-build build-essential pkg-config curl bc jq \ + libslirp-dev libfdt-dev libglib2.0-dev libssl-dev libpixman-1-dev \ + flex bison + +git clone --depth 1 --branch $QEMU_TAG https://gitlab.com/qemu-project/qemu.git +pushd qemu +./configure --target-list=riscv64-softmmu --prefix=$OUTPUT && \ + make -j$(nproc) && make install +popd diff --git a/riscv64/start_in_qemu.sh b/riscv64/start_in_qemu.sh new file mode 100755 index 0000000..599b914 --- /dev/null +++ b/riscv64/start_in_qemu.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ex + +$QEMU_DIR/bin/qemu-system-riscv64 \ + -M virt,aclint=on,aia=aplic-imsic -nographic \ + -smp 3 -m 6G \ + -bios $OPENSBI_DIR/fw_jump.elf \ + -kernel $KERNEL_DIR/Image \ + -device virtio-net-device,netdev=usernet -netdev user,id=usernet,hostfwd=tcp::2222-:22 \ + -virtfs local,path=$ROOTFS_DIR,mount_tag=rootfs,security_model=none,id=rootfs \ + -append "root=rootfs rw rootfstype=9p rootflags=trans=virtio,cache=mmap,msize=512000 console=ttyS0 earlycon=sbi nokaslr rdinit=/sbin/init" 2>&1 & + +# Copy WORKDIR to rootfs +cp -a $WORKDIR /opt/rootfs/root + +HOST=riscv-qemu + +while ! nc -z localhost 2222; do + echo "Dialing qemu-system-riscv64..." + sleep 1 +done + +# Issue command +COMMAND=$@ +ssh $HOST "export PATH=\"\$PATH:/root/.cargo/bin\" && cd workdir && $COMMAND"