From d1a20e58b68dde9ff2c88ae96eb33a67e08527a6 Mon Sep 17 00:00:00 2001 From: ASleepyCat Date: Sun, 6 Apr 2025 15:29:02 +0800 Subject: [PATCH] Add CI and improve build system (#293) ## Link to GitHub Issue, if one exists Fixes #292 ## Description of change This PR adds CI with Github Actions. Runs on every push and pull request. If a new commit is pushed, any previous runs are cancelled for that branch. In total, these optimisations (on a 7800X3D with 32GB 6000MT/s RAM and an NVMe Gen4 SSD): * Reduces `spicetools/deps` image size by around 200 MB, and build time by almost 50% * Reduces compilation time by around 9% Changes: * Pull archlinux image from GHCR so that we don't run into pull limits from docker.io * Use `base-devel` archlinux image * Reduces image build time from 141 seconds to 109 seconds on my machine * Clean up unneeded packages * Small reduction in `spicetools/deps` image size (4.35 GB vs 4.14 GB) * Use precompiled `yay` binary * Reduces image build time from 163 seconds to 141 seconds on my machine * Use `nproc` to get the number of cores instead of parsing `/proc/cpuinfo` with `awk` * Use `ninja` to compile * Speeds up compile times a little, around 13 seconds on my machine * Don't run `docker run` in interactive TTY in `build_docker.sh` * Github Actions disallows interactive TTYs * Still allowed for the batch script * `time` the compilation process * Enable BuildKit in the bash script * Required for build contexts * Don't use `USER` instruction * Breaks Github Actions, doesn't seem to affect build? * More info: https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user * `USER` also seems to cause issues on a mounted ReFS image * Add executable bits to bash scripts Future ideas: * Combine `external/Dockerfile` and `./Dockerfile` into one to deduplicate images and save disk space * Fix the weird `/src/src` directory structure in the Dockerfile and build scripts * Investigate CMake and ninja optimisations? ## Compiling Check CI status on my fork. ## Testing Tested with docker on Windows. Also play tested a bit to make sure the Github Actions artifact works. --- .github/workflows/ci.yml | 22 ++++++++++ src/spice2x/Dockerfile | 7 ++-- src/spice2x/build_all.sh | 56 +++++++++++++------------- src/spice2x/build_docker.sh | 4 +- src/spice2x/external/docker/Dockerfile | 17 ++------ 5 files changed, 59 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/ci.yml mode change 100644 => 100755 src/spice2x/build_all.sh mode change 100644 => 100755 src/spice2x/build_docker.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d372986 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +on: [push, pull_request] +name: Continuous Integration +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + fw-ci: + name: Build + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./src/spice2x + steps: + - uses: actions/checkout@v4 + - name: Compile + run: ./build_docker.sh + - uses: actions/upload-artifact@v4 + with: + name: spice2x + path: src/spice2x/bin + if-no-files-found: error \ No newline at end of file diff --git a/src/spice2x/Dockerfile b/src/spice2x/Dockerfile index 053d3e8..fc0e36a 100644 --- a/src/spice2x/Dockerfile +++ b/src/spice2x/Dockerfile @@ -1,8 +1,7 @@ FROM spicetools/deps WORKDIR /src -RUN chown user:user /src -USER user -COPY --chown=user:user --from=gitroot . /src/.git -COPY --chown=user:user . /src/src/spice2x + +COPY --from=gitroot . /src/.git +COPY . /src/src/spice2x WORKDIR /src/src/spice2x CMD ["./build_all.sh"] diff --git a/src/spice2x/build_all.sh b/src/spice2x/build_all.sh old mode 100644 new mode 100755 index eb0c915..522a19f --- a/src/spice2x/build_all.sh +++ b/src/spice2x/build_all.sh @@ -57,7 +57,7 @@ then fi # determine number of cores -CORES=$(awk '/^processor\t/ {cores[$NF]++} END {print length(cores)}' /proc/cpuinfo) +CORES=$(nproc) # print information echo "" @@ -74,34 +74,36 @@ echo "Build Type: $BUILD_TYPE" echo "Cores: $CORES" echo "" -# 32 bit -echo "Building 32bit targets..." -echo "=========================" -if ((CLEAN_BUILD > 0)) -then - rm -rf ${BUILDDIR_32} -fi -mkdir -p ${BUILDDIR_32} -pushd ${BUILDDIR_32} > /dev/null -cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_32} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} $OLDPWD && make -j ${CORES} ${TARGETS_32} -popd > /dev/null +time ( + # 32 bit + echo "Building 32bit targets..." + echo "=========================" + if ((CLEAN_BUILD > 0)) + then + rm -rf ${BUILDDIR_32} + fi + mkdir -p ${BUILDDIR_32} + pushd ${BUILDDIR_32} > /dev/null + cmake -G "Ninja" -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_32} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} $OLDPWD && ninja ${TARGETS_32} + popd > /dev/null -# 64 bit -echo "" -echo "Building 64bit targets..." -echo "=========================" -if ((CLEAN_BUILD > 0)) -then - rm -rf ${BUILDDIR_64} -fi -mkdir -p ${BUILDDIR_64} -pushd ${BUILDDIR_64} > /dev/null -cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_64} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} $OLDPWD && make -j ${CORES} ${TARGETS_64} -popd > /dev/null + # 64 bit + echo "" + echo "Building 64bit targets..." + echo "=========================" + if ((CLEAN_BUILD > 0)) + then + rm -rf ${BUILDDIR_64} + fi + mkdir -p ${BUILDDIR_64} + pushd ${BUILDDIR_64} > /dev/null + cmake -G "Ninja" -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_64} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} $OLDPWD && ninja ${TARGETS_64} + popd > /dev/null -echo "" -echo "Compilation process done :)" -echo "===========================" + echo "" + echo "Compilation process done :)" + echo "===========================" +) # generate PDBs if false # ((DEBUG > 0)) diff --git a/src/spice2x/build_docker.sh b/src/spice2x/build_docker.sh old mode 100644 new mode 100755 index f4ab350..9147847 --- a/src/spice2x/build_docker.sh +++ b/src/spice2x/build_docker.sh @@ -1,6 +1,4 @@ #!/bin/bash -export DOCKER_BUILDKIT=0 - docker build --pull $PWD/external/docker -t spicetools/deps --platform linux/x86_64 docker build --build-context gitroot=$PWD/../../.git . -t spicetools/spice:latest --no-cache -docker run --rm -it -v $PWD/dist:/src/src/spice2x/dist -v $PWD/bin:/src/src/spice2x/bin spicetools/spice +docker run --rm -v $PWD/dist:/src/src/spice2x/dist -v $PWD/bin:/src/src/spice2x/bin spicetools/spice diff --git a/src/spice2x/external/docker/Dockerfile b/src/spice2x/external/docker/Dockerfile index 392f152..1ef70c4 100644 --- a/src/spice2x/external/docker/Dockerfile +++ b/src/spice2x/external/docker/Dockerfile @@ -1,20 +1,11 @@ -FROM archlinux:base +FROM ghcr.io/archlinux/archlinux:base-devel -RUN pacman --noconfirm -Syu gettext \ - base-devel \ - git \ +RUN pacman --noconfirm -Syu git \ bash \ zip \ upx \ - sudo \ - binutils \ - file \ - make \ - gcc \ - fakeroot \ - diffutils \ + ninja \ cmake \ - awk \ unzip \ mingw-w64-crt \ mingw-w64-winpthreads \ @@ -22,4 +13,4 @@ RUN pacman --noconfirm -Syu gettext \ mingw-w64-headers \ mingw-w64-binutils RUN useradd user -m && echo "ALL ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -RUN su user -c "cd /tmp/ && git clone https://aur.archlinux.org/yay.git --depth=1 && cd yay && makepkg --noconfirm -si && yay --noconfirm -S mingw-w64-cmake" +RUN su user -c "cd /tmp/ && git clone https://aur.archlinux.org/yay-bin.git --depth=1 && cd yay-bin && makepkg --noconfirm -si && yay --noconfirm -S mingw-w64-cmake"