From 0da7ddb738df60b3650d36832c9ede040a7ba6b9 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Wed, 22 Jun 2022 06:36:35 -0400 Subject: concurrency and caching (#1) * use concurrency * run here too * allow test failure * only push on push * fix coresight mode build * support but disable coresight * woops * no unicorn for arm * update codeql and fix makefile * forgot those * disable forward cache * fix needing buildx Co-authored-by: Ruben ten Hove --- .dockerignore | 4 +- .github/workflows/build-push-test-container.yaml | 61 --------------- .github/workflows/build-test-push-container.yaml | 96 ++++++++++++++++++++++++ .github/workflows/codeql-analysis.yml | 6 +- Dockerfile | 16 ++-- GNUmakefile | 26 +++++-- 6 files changed, 134 insertions(+), 75 deletions(-) delete mode 100644 .github/workflows/build-push-test-container.yaml create mode 100644 .github/workflows/build-test-push-container.yaml diff --git a/.dockerignore b/.dockerignore index d05bf1c6..9d3d434b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +/.github .test .test2 .sync_tmp @@ -10,6 +11,7 @@ ld in out core* +!/coresight_mode afl-analyze afl-as afl-clang @@ -62,4 +64,4 @@ examples/afl_network_proxy/afl-network-client examples/afl_frida/afl-frida examples/afl_frida/libtestinstr.so examples/afl_frida/frida-gum-example.c -examples/afl_frida/frida-gum.h \ No newline at end of file +examples/afl_frida/frida-gum.h diff --git a/.github/workflows/build-push-test-container.yaml b/.github/workflows/build-push-test-container.yaml deleted file mode 100644 index e2b828ac..00000000 --- a/.github/workflows/build-push-test-container.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: Build, test and push container image - -on: - push: - branches: - - stable - - dev - tags: - - '*' - -jobs: - build: - name: Build, test and push container image - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build amd64 image - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64 - load: true - tags: aflplusplus/aflplusplus:test - - name: Build arm64 image - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/arm64 - load: true - tags: aflplusplus/aflplusplus:test - - name: Test linux/amd64 image - run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests - - name: Test linux/arm64 image - run: docker run --rm --platform linux/arm64 aflplusplus/aflplusplus:test make tests - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Publish ${{ github.ref_name }} to docker.io registry - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: aflplusplus/aflplusplus:${{ github.ref_name }} - if: ${{ github.ref_name != 'stable' }} - - name: Publish stable and latest to docker.io registry - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest - if: ${{ github.ref_name == 'stable' }} diff --git a/.github/workflows/build-test-push-container.yaml b/.github/workflows/build-test-push-container.yaml new file mode 100644 index 00000000..5d9e69e4 --- /dev/null +++ b/.github/workflows/build-test-push-container.yaml @@ -0,0 +1,96 @@ +name: Build and test container image +on: + push: + branches: + - stable + - dev + tags: + - '*' + pull_request: + branch: + - stable + - dev + +jobs: + build-amd64: + name: Build and test amd64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build amd64 + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + load: true + tags: aflplusplus/aflplusplus:test + # cache-from: type=gha # Ensure we always build a fresh image. We just use the cache for the subsequent push job. + cache-to: type=gha,mode=max + - name: Test linux/amd64 image + run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests + continue-on-error: true + build-arm64: + name: Build and test arm64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build arm64 + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/arm64 + load: true + tags: aflplusplus/aflplusplus:test + # cache-from: type=gha # Ensure we always build a fresh image. We just use the cache for the subsequent push job. + cache-to: type=gha,mode=max + - name: Test linux/arm64 image + run: docker run --rm --platform linux/arm64 aflplusplus/aflplusplus:test make tests + continue-on-error: true + push: + needs: + - build-amd64 + - build-arm64 + if: ${{ github.event_name == 'push' }} + name: Push image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Publish ${{ github.ref_name }} for amd64 and arm64 to docker.io registry + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: aflplusplus/aflplusplus:${{ github.ref_name }} + cache-from: type=gha + # cache-to: type=gha,mode=max # No need to add to cache as we'll never use this + if: ${{ github.ref_name != 'stable' }} + - name: Publish stable and latest to docker.io registry + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest + cache-from: type=gha + # cache-to: type=gha,mode=max # No need to add to cache as we'll never use this + if: ${{ github.ref_name == 'stable' }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d5dce05f..55d477fc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -21,12 +21,12 @@ jobs: uses: actions/checkout@v3 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/Dockerfile b/Dockerfile index e9b1bc7e..b8b45ac5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,9 +33,9 @@ ENV PATH=$PATH:/etc/cargo/bin ARG LLVM_VERSION=14 ARG GCC_VERSION=12 -RUN mkdir -p /usr/local/share/keyrings && \ - echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ - wget -qO /usr/local/share/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key +RUN mkdir -p /etc/apt/keyrings && \ + echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ + wget -qO /etc/apt/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key RUN apt-get update && \ apt-get -y install --no-install-recommends \ @@ -45,9 +45,11 @@ RUN apt-get update && \ libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \ libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ - $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) && \ + $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) \ + $([ "$(dpkg --print-architecture)" = "arm64" ] && echo libcapstone-dev) && \ rm -rf /var/lib/apt/lists/* - # arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86 + # gcc-multilib is only used for -m32 support on x86 + # libcapstone-dev is used for coresight_mode on arm64 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 @@ -66,6 +68,10 @@ COPY . . # Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029 ENV NO_NYX 1 +# Build currently broken +ENV NO_CORESIGHT 1 +ENV NO_UNICORN_ARM64 1 + RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && make clean && \ make distrib && make install && make clean diff --git a/GNUmakefile b/GNUmakefile index fad619cf..9982ad75 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -623,15 +623,23 @@ endif $(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT $(MAKE) -C coresight_mode + endif endif ifeq "$(SYS)" "Linux" -ifndef NO_NYX + ifndef NO_NYX cd nyx_mode && ./build_nyx_support.sh -endif + endif endif cd qemu_mode && sh ./build_qemu_support.sh + ifeq "$(ARCH)" "aarch64" + ifndef NO_UNICORN_ARM64 cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif + else + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif endif .PHONY: binary-only @@ -645,15 +653,23 @@ binary-only: test_shm test_python ready $(PROGS) $(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT $(MAKE) -C coresight_mode + endif endif ifeq "$(SYS)" "Linux" ifndef NO_NYX - -cd nyx_mode && ./build_nyx_support.sh + cd nyx_mode && ./build_nyx_support.sh endif endif - -cd qemu_mode && sh ./build_qemu_support.sh - -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + cd qemu_mode && sh ./build_qemu_support.sh + ifeq "$(ARCH)" "aarch64" + ifndef NO_UNICORN_ARM64 + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif + else + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif endif .PHONY: source-only -- cgit 1.4.1