diff options
| author | Martin Nowack <m.nowack@imperial.ac.uk> | 2023-02-23 22:01:12 +0000 |
|---|---|---|
| committer | Cristian Cadar <c.cadar@imperial.ac.uk> | 2023-03-17 22:38:16 +0000 |
| commit | ca60811f5d55254bec378919430dd0e7e6e5cdd1 (patch) | |
| tree | dbd4f7a78929eb959e26755c22e5157808a09862 /scripts | |
| parent | dd492f8763f13312c17eb67af33e3e90217a30e1 (diff) | |
| download | klee-ca60811f5d55254bec378919430dd0e7e6e5cdd1.tar.gz | |
Update build scripts
* Support for Ubuntu 22.04 * Remove support for Python2 * Better separation between sanitizer builds and non-sanitizer builds * Fix build of metaSMT on newer Ubuntu versions * Use ninja to build LLVM * Simplifying building arbitrary LLVM configurations, e.g. different LLVM sanitizer builds (MemSan, UBSan, ASan) * Use MemSan with origin tracking * Build sqlite3 container correctly * Add support to provide sqlite3 version number
Diffstat (limited to 'scripts')
31 files changed, 641 insertions, 311 deletions
diff --git a/scripts/build/d-klee-linux-ubuntu.inc b/scripts/build/d-klee-linux-ubuntu.inc index 16db5cdf..30526bb5 100644 --- a/scripts/build/d-klee-linux-ubuntu.inc +++ b/scripts/build/d-klee-linux-ubuntu.inc @@ -1,7 +1,7 @@ # TODO remove adding sudo package # Create ``klee`` user for container with password ``klee``. # and give it password-less sudo access (temporarily so we can use the CI scripts) -RUN apt update && DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install sudo emacs-nox vim-nox file python3-dateutil && \ +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install sudo emacs-nox vim-nox file python3-dateutil && \ rm -rf /var/lib/apt/lists/* && \ useradd -m klee && \ echo klee:klee | chpasswd && \ @@ -11,19 +11,23 @@ RUN apt update && DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends # Copy across source files needed for build COPY --chown=klee:klee . /tmp/klee_src/ +USER klee +WORKDIR /home/klee # Build and set klee user to be owner -RUN /tmp/klee_src/scripts/build/build.sh --debug --install-system-deps klee && chown -R klee:klee /tmp/klee_build* && pip3 install flask wllvm && \ - rm -rf /var/lib/apt/lists/* +RUN /tmp/klee_src/scripts/build/build.sh --debug --install-system-deps klee && pip3 install flask wllvm && \ + sudo rm -rf /var/lib/apt/lists/* + -ENV PATH="$PATH:/tmp/llvm-60-install_O_D_A/bin:/home/klee/klee_build/bin" +ENV PATH="$PATH:/tmp/llvm-110-install_O_D_A/bin:/home/klee/klee_build/bin:/home/klee/.local/bin" ENV BASE=/tmp -USER klee -WORKDIR /home/klee +# Add KLEE header files to system standard include folder +RUN sudo /bin/bash -c 'ln -s /tmp/klee_src/include/klee /usr/include/' + ENV LD_LIBRARY_PATH /home/klee/klee_build/lib/ # Add KLEE binary directory to PATH RUN /bin/bash -c 'ln -s ${BASE}/klee_src /home/klee/ && ln -s ${BASE}/klee_build* /home/klee/klee_build' # TODO Remove when STP is fixed -RUN /bin/bash -c 'echo "export LD_LIBRARY_PATH=$(cd ${BASE}/metaSMT-*-deps/stp-git-basic/lib/ && pwd):$LD_LIBRARY_PATH" >> /home/klee/.bashrc' +RUN /bin/bash -c 'echo "export LD_LIBRARY_PATH=$(cd ${BASE}/metaSMT-*-deps/stp-git-basic/lib/ && pwd):$LD_LIBRARY_PATH" >> /home/klee/.bashrc' \ No newline at end of file diff --git a/scripts/build/p-clang-linux-ubuntu-22.04.inc b/scripts/build/p-clang-linux-ubuntu-22.04.inc new file mode 100644 index 00000000..dc125bd5 --- /dev/null +++ b/scripts/build/p-clang-linux-ubuntu-22.04.inc @@ -0,0 +1,8 @@ +get_docker_config_id_clang() { + if [[ "${LLVM_VERSION_SHORT}" -ge "11" ]]; then + echo "" + return 0; + else + return 1; + fi +} diff --git a/scripts/build/p-clang-linux-ubuntu.inc b/scripts/build/p-clang-linux-ubuntu.inc index 286246ca..85f02c14 100644 --- a/scripts/build/p-clang-linux-ubuntu.inc +++ b/scripts/build/p-clang-linux-ubuntu.inc @@ -3,40 +3,59 @@ install_binary_artifact_clang() { local version="-${LLVM_VERSION_MAJOR}" source "${DIR}/common-functions" + with_sudo apt-get update -y + + # Check if package in standard repository otherwise use upstream + if ! apt-cache show "llvm${version}"; then + if [[ -z "$(which wget)" ]]; then + # Add certificate + with_sudo apt-get update -y + dependencies=( + ca-certificates + wget + lsb-release + gnupg + ) + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" + fi - # Add certificate - with_sudo apt update -y - dependencies=( - ca-certificates - wget - lsb-release - gnupg - ) - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" - - # Add LLVM upstream repository if available - codename="$(lsb_release --codename --short)" - if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add - - - apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main" - if ! grep -rq "${apt_entry}" /etc/apt; then - echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list - with_sudo apt update -y + # Add LLVM upstream repository if available + codename="$(lsb_release --codename --short)" + if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then + apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main" + if ! grep -rq "${apt_entry}" /etc/apt; then + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add - + echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list + with_sudo apt-get update -y + fi fi fi - - with_sudo apt update -y dependencies=( "llvm${version}" "clang${version}" ) #Install essential dependencies - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" || return 1 + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" || return 1 } -get_docker_config_id_clang() { - return 0 -} +setup_artifact_variables_clang() { + local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}" + local version="-${LLVM_VERSION_MAJOR}" + + # Only set LLVM_CONFIG if not set yet + if [[ -z "${LLVM_CONFIG:-}" ]]; then + LLVM_CONFIG=$(which "llvm-config${version}") + fi + + local bin_path="" + bin_path=$(which "clang${version}") + [[ -z "${bin_path}" ]] && return 1 + + bin_path="$(dirname "$(readlink -f "${bin_path}")")" + [[ -z "${bin_path}" ]] && return 1 + + BITCODE_CC="${bin_path}/clang" + BITCODE_CXX="${bin_path}/clang++" +} \ No newline at end of file diff --git a/scripts/build/p-clang-linux.inc b/scripts/build/p-clang-linux.inc index 6481fdd8..519333c6 100644 --- a/scripts/build/p-clang-linux.inc +++ b/scripts/build/p-clang-linux.inc @@ -7,9 +7,12 @@ setup_artifact_variables_clang() { LLVM_CONFIG=$(which "llvm-config${version}") fi - local bin_path="" - bin_path="$(dirname "$(readlink -f "$(which "clang${version}")")")" + bin_path=$(which "clang${version}") + [[ -z "${bin_path}" ]] && return 1 + + bin_path="$(dirname "$(readlink -f "${bin_path}")")" + [[ -z "${bin_path}" ]] && return 1 BITCODE_CC="${bin_path}/clang" BITCODE_CXX="${bin_path}/clang++" diff --git a/scripts/build/p-clang.inc b/scripts/build/p-clang.inc new file mode 100644 index 00000000..1b4593c9 --- /dev/null +++ b/scripts/build/p-clang.inc @@ -0,0 +1,62 @@ +local_setup_build_variables_clang() { + source "${DIR}/common-functions" + LLVM_SUFFIX="" + + LLVM_SUFFIX+="_O" + LLVM_SUFFIX+="_ND" + LLVM_SUFFIX+="_NA" + LLVM_SUFFIX+="_RTTI" + + LLVM_SRC_BASE="${BASE}/llvm-${LLVM_VERSION_SHORT}" + LLVM_BUILD="${LLVM_SRC_BASE}-build${LLVM_SUFFIX}" + LLVM_INSTALL="${LLVM_SRC_BASE}-install${LLVM_SUFFIX}" + LLVM_BIN="${LLVM_INSTALL}/bin" + LLVM_BUILD_BIN="${LLVM_BUILD}/bin" + + LLVM_CONFIG="${LLVM_BIN}/llvm-config" + BITCODE_CC="${LLVM_BIN}/clang" + BITCODE_CXX="${LLVM_BIN}/clang++" +} + +get_docker_config_id_clang() { + ( + local_setup_build_variables_clang + echo "${LLVM_VERSION_SHORT}${LLVM_SUFFIX}" + ) +} + +get_build_artifacts_clang() { + ( + local_setup_build_variables_clang + echo "${LLVM_INSTALL}" + ) +} + +# Check if the binary artifact is installed +is_installed_clang() { + local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}" + local version="${LLVM_VERSION_MAJOR}" + + ( + local_setup_build_variables_clang + # Check if clang is installed at a potential location + [[ -f "${LLVM_BIN}/clang" ]] + ) +} + +setup_artifact_variables_clang() { + is_installed_clang || return 1 + + local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}" + local version="${LLVM_VERSION_MAJOR}" + local_setup_build_variables_clang + echo "${LLVM_INSTALL}" +} + +install_build_dependencies_clang() { + return 0 +} + +setup_build_variables_clang() { + return 0 +} \ No newline at end of file diff --git a/scripts/build/p-klee-linux-ubuntu.inc b/scripts/build/p-klee-linux-ubuntu.inc index c2a0808c..8bf39a19 100644 --- a/scripts/build/p-klee-linux-ubuntu.inc +++ b/scripts/build/p-klee-linux-ubuntu.inc @@ -12,6 +12,12 @@ install_build_dependencies_klee() { python3-wheel ) + if [[ "${SOLVERS:-}" == "metaSMT" ]]; then + dependencies+=( + libboost-dev + libgmp-dev + ) + fi if [[ $(to_bool "${COVERAGE}") -eq 1 ]]; then dependencies+=(lcov curl git) fi @@ -20,7 +26,7 @@ install_build_dependencies_klee() { dependencies+=(doxygen graphviz) fi - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" pip3 install --user lit pip3 install --user tabulate diff --git a/scripts/build/p-klee.inc b/scripts/build/p-klee.inc index a69f654a..82dedeaa 100644 --- a/scripts/build/p-klee.inc +++ b/scripts/build/p-klee.inc @@ -17,9 +17,7 @@ build_klee() { CMAKE_PREFIX_PATH=("") local CMAKE_ARGUMENTS=( - "-DLLVM_CONFIG_BINARY=${LLVM_CONFIG}" - "-DLLVMCC=${BITCODE_CC}" - "-DLLVMCXX=${BITCODE_CXX}" + "-DLLVM_DIR=${LLVM_INSTALL}" "-DGTEST_SRC_DIR=${GTEST_INSTALL_PATH}" "-DENABLE_UNIT_TESTS=TRUE" "-DENABLE_SYSTEM_TESTS=TRUE" @@ -160,12 +158,18 @@ fi CXX_FLAGS+=("${SANITIZER_CXX_FLAGS[@]}") LD_FLAGS+=("${SANITIZER_LD_FLAGS[@]}") + SANITIZER_DIR="$(cd "$(dirname "${SANITIZER_BITCODE_CC}")" && pwd)" + CMAKE_ARGUMENTS+=( "-DCMAKE_C_COMPILER=${SANITIZER_C_COMPILER}" "-DCMAKE_CXX_COMPILER=${SANITIZER_CXX_COMPILER}" ) fi + CMAKE_ARGUMENTS+=( + "-DLLVMCC=${BITCODE_CC}" + "-DLLVMCXX=${BITCODE_CXX}" + ) mkdir -p "${KLEE_BUILD_DIR}" || return 1 diff --git a/scripts/build/p-libcxx.inc b/scripts/build/p-libcxx.inc index 3ee41660..641fad61 100644 --- a/scripts/build/p-libcxx.inc +++ b/scripts/build/p-libcxx.inc @@ -107,6 +107,7 @@ get_build_artifacts_libcxx() { ( setup_build_variables_libcxx echo "${LIBCXX_INSTALL}" + echo "${LIBCXX_SRC}" ) } diff --git a/scripts/build/p-llvm-linux-ubuntu.inc b/scripts/build/p-llvm-linux-ubuntu.inc index 48a0083f..1af095f9 100644 --- a/scripts/build/p-llvm-linux-ubuntu.inc +++ b/scripts/build/p-llvm-linux-ubuntu.inc @@ -12,12 +12,12 @@ install_build_dependencies_llvm() { groff gcc g++ - python-dev python3-distutils make git # To check out code zlib1g-dev git + ninja-build ) if [[ "${SANITIZERS[*]}" == "memory" ]]; then @@ -31,11 +31,15 @@ install_build_dependencies_llvm() { install_binary_artifact_llvm() { # No need to check for optimised, we can build against LLVM with optimised and non-optimised versions # local enable_optimized=$(to_bool "${ENABLE_OPTIMIZED}") - local enable_debug=$(to_bool "${ENABLE_DEBUG}") - local disable_assertions=$(to_bool "${DISABLE_ASSERTIONS}") - local requires_rtti=$(to_bool "${REQUIRES_RTTI}") + local enable_debug + enable_debug=$(to_bool "${ENABLE_DEBUG}") + local disable_assertions + disable_assertions=$(to_bool "${DISABLE_ASSERTIONS}") + local requires_rtti + requires_rtti=$(to_bool "${REQUIRES_RTTI}") local LLVM_VERSION_MAJOR="${LLVM_VERSION/.*/}" - local LLVM_VERSION_MINOR="${LLVM_VERSION/*./}" + local version="${LLVM_VERSION_MAJOR}" + # No support for LLVM packages with debug information, incompatible if requested otherwise @@ -50,27 +54,30 @@ install_binary_artifact_llvm() { # Enable/Disable optimized does not matter source "${DIR}/common-functions" - # Add certificate - with_sudo apt update -y - dependencies=( - ca-certificates - wget - lsb-release - gnupg - ) - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" - - local version="-${LLVM_VERSION_MAJOR}" - - # Add LLVM upstream repository if available - codename="$(lsb_release --codename --short)" - if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add - + # Check if package in standard repository otherwise use upstream + with_sudo apt-get update -y + if ! apt-cache show "llvm${version}"; then + if [[ -z "$(which wget)" ]]; then + # Add certificate + with_sudo apt-get update -y + dependencies=( + ca-certificates + wget + lsb-release + gnupg + ) + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" + fi - apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main" - if ! grep -rq "${apt_entry}" /etc/apt; then - echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list - with_sudo apt update -y + # Add LLVM upstream repository if available + codename="$(lsb_release --codename --short)" + if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then + apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main" + if ! grep -rq "${apt_entry}" /etc/apt; then + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add - + echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list + with_sudo apt-get update -y + fi fi fi @@ -82,7 +89,7 @@ install_binary_artifact_llvm() { ) #Install essential dependencies - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" || return 1 + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" || return 1 } check_llvm_config_version() { @@ -92,7 +99,7 @@ check_llvm_config_version() { lc="$2" # path to llvm-config # If not set return error - [[ -z "${lc}" ]] && return 1 + [[ -n "${lc}" ]] || return 1 # First check, if the provided llvm-config is a full path if [[ ! -f "${lc}" ]]; then @@ -140,9 +147,6 @@ is_installed_llvm() { lc=$(which "llvm-config-${LLVM_VERSION_MAJOR}") check_llvm_config_version 1 "${lc}" && return 0 - # As alternative, try the version-less number - lc=$(which "llvm-config") - check_llvm_config_version 1 "${lc}" && return 0 return 1 } @@ -155,22 +159,22 @@ setup_artifact_variables_llvm() { local lc="" # Check for llvm-config without suffix but correct version number lc=$(which "llvm-config") - local is_ins=$(check_llvm_config_version 1 "${lc}") + local is_ins + is_ins=$(check_llvm_config_version 1 "${lc}") if [[ ! "${is_ins}" ]]; then # Check if llvm-config with the right version exists - lc=$(which "llvm-config-${version}") + lc=$(which "llvm-config-${version}") || return 1 is_ins=$(check_llvm_config_version 1 "${lc}") || return 1 fi LLVM_CONFIG="${lc}" - LLVM_INSTALL="$(${lc} --bindir)" - BITCODE_CC="${LLVM_INSTALL}/clang" - BITCODE_CXX="${LLVM_INSTALL}/clang++" - LLVM_BIN="${LLVM_INSTALL}/bin" + LLVM_INSTALL="$(${lc} --prefix)" + LLVM_BIN="$(${lc} --bindir)" + BITCODE_CC="${LLVM_BIN}/clang" + BITCODE_CXX="${LLVM_BIN}/clang++" } get_build_artifacts_llvm() { - is_installed_llvm - [[ $? -ne 0 ]] && return 1 + is_installed_llvm || return 1 return 0 -} +} \ No newline at end of file diff --git a/scripts/build/p-llvm.inc b/scripts/build/p-llvm.inc index d1b9374e..abf895ae 100644 --- a/scripts/build/p-llvm.inc +++ b/scripts/build/p-llvm.inc @@ -55,38 +55,14 @@ setup_build_variables_llvm() { LLVM_BIN="${LLVM_INSTALL}/bin" LLVM_BUILD_BIN="${LLVM_BUILD}/bin" + if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then + LLVM_BUILD_LIBCXX="${LLVM_INSTALL}-libcxx" + fi + + LLVM_CONFIG="${LLVM_BIN}/llvm-config" BITCODE_CC="${LLVM_BIN}/clang" BITCODE_CXX="${LLVM_BIN}/clang++" - - for sanitizer in "${SANITIZERS[@]}"; do - [[ -z "${sanitizer}" ]] && continue - # Undefined Behaviour Sanitizer - if [ "${sanitizer}" == "memory" ]; then - SANITIZER_LLVM_UNINSTRUMENTED="${LLVM_BUILD}_uninstrumented" - SANITIZER_LLVM_LIBCXX="${LLVM_BUILD}_libcxx" - - MSAN_LINK_FLAGS=("-lc++abi" "-Wl,--rpath=${SANITIZER_LLVM_LIBCXX}/lib" "-L${SANITIZER_LLVM_LIBCXX}/lib") - MSAN_FLAGS=("${MSAN_LINK_FLAGS[@]}" -nostdinc++ - -isystem "${SANITIZER_LLVM_LIBCXX}/include" - -isystem "${SANITIZER_LLVM_LIBCXX}/include/c++/v1" - "-fsanitize=memory" - "-fsanitize-memory-track-origins" - -w -fno-omit-frame-pointer -g) - SANITIZER_CXX_FLAGS+=("${MSAN_FLAGS[@]}" "-stdlib=libc++") - SANITIZER_C_FLAGS+=("${MSAN_FLAGS[@]}") - SANITIZER_LD_FLAGS+=("${MSAN_LINK_FLAGS[@]}") - SANITIZER_C_COMPILER="${SANITIZER_LLVM_UNINSTRUMENTED}/bin/clang" - SANITIZER_CXX_COMPILER="${SANITIZER_LLVM_UNINSTRUMENTED}/bin/clang++" - - # Use the uninstrumented compiler - BITCODE_CC="${SANITIZER_C_COMPILER}" - BITCODE_CXX="${SANITIZER_CXX_COMPILER}" - # But point to the instrumented llvm-config - LLVM_CONFIG="${LLVM_BIN}/llvm-config" - continue - fi - done } download_llvm() { @@ -117,71 +93,44 @@ configure_llvm() { local disable_assertions=$(to_bool "${DISABLE_ASSERTIONS}") local requires_rtti=$(to_bool "${REQUIRES_RTTI}") - # For memory sanitizer, we have a multi-stage build process - if [[ "${SANITIZER_BUILD}" == "memory" ]]; then - # Build uninstrumented compiler - mkdir -p "${SANITIZER_LLVM_UNINSTRUMENTED}" - cd "${SANITIZER_LLVM_UNINSTRUMENTED}" - cmake -GNinja -DCMAKE_BUILD_TYPE=Release \ - "-DLLVM_OPTIMIZED_TABLEGEN=ON" \ - "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi" \ - "-DLLVM_TARGETS_TO_BUILD=X86" \ - "${LLVM_SRC_BASE}/llvm" - ninja compiler-rt cxx cxxabi clang || return 1 - - # Build instrumented libc/libc++ - mkdir -p "${SANITIZER_LLVM_LIBCXX}" - cd "${SANITIZER_LLVM_LIBCXX}" - cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi" \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - "-DCMAKE_C_COMPILER=${SANITIZER_C_COMPILER}" \ - "-DCMAKE_CXX_COMPILER=${SANITIZER_CXX_COMPILER}" \ - "-DLLVM_OPTIMIZED_TABLEGEN=ON" \ - "-DLLVM_TARGETS_TO_BUILD=X86" \ - "${LLVM_SRC_BASE}/llvm" - ninja compiler-rt cxx cxxabi || return 1 - - # Build instrumented clang - mkdir -p "${LLVM_BUILD}" - cd "${LLVM_BUILD}" - C_F="${SANITIZER_C_FLAGS[*]}" - CXX_F="${SANITIZER_CXX_FLAGS[*]}" - LD_F="${SANITIZER_LD_FLAGS[*]}" - cmake -GNinja \ - "-DCMAKE_C_COMPILER=${SANITIZER_C_COMPILER}" \ - "-DCMAKE_CXX_COMPILER=${SANITIZER_CXX_COMPILER}" \ - -DCMAKE_C_FLAGS="$C_F" \ - -DCMAKE_CXX_FLAGS="${CXX_F}" \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi" \ - -DLLVM_ENABLE_ASSERTIONS=On \ - -DLLVM_USE_SANITIZER=MemoryWithOrigins \ - -DLLVM_ENABLE_LIBCXX=ON \ - -DCMAKE_EXE_LINKER_FLAGS="${LD_F}" \ - -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL}" \ - -DBUILD_SHARED_LIBS=ON \ - -DLLVM_OPTIMIZED_TABLEGEN=ON \ - -DLLVM_TARGETS_TO_BUILD=X86 \ - "${LLVM_SRC_BASE}/llvm" - return 0 - fi - # Configure; build; and install mkdir -p "${LLVM_BUILD}" - cd "${LLVM_BUILD}" - - # Skip building if already finished - [[ -e "${LLVM_BUILD}/.build_finished" ]] && return 0 # Configure LLVM CONFIG=( "-DCMAKE_INSTALL_PREFIX=${LLVM_INSTALL}" - "-DLLVM_BUILD_LLVM_DYLIB=TRUE" ) + + if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then + # Two stage process, build libcxx and libcxxabi + + # Skip building if already finished + if [[ ! -f "${LLVM_BUILD_LIBCXX}/.build_finished" ]]; then + mkdir -p "${LLVM_BUILD_LIBCXX}" + cd "${LLVM_BUILD_LIBCXX}" + cmake -GNinja "${LLVM_SRC_BASE}/llvm" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \ + -DCMAKE_C_COMPILER="${SANITIZER_C_COMPILER}" \ + -DCMAKE_CXX_COMPILER="${SANITIZER_CXX_COMPILER}" \ + -DLLVM_USE_SANITIZER=MemoryWithOrigins + cmake --build . -- cxx cxxabi || return 1 + touch "${LLVM_BUILD_LIBCXX}/.build_finished" + fi + + CONFIG+=( + -DCMAKE_CXX_FLAGS="-fsanitize=memory -stdlib=libc++ -isystem ${LLVM_BUILD_LIBCXX}/include -isystem ${LLVM_BUILD_LIBCXX}/include/c++/v1" + ) + + LLVM_LINKER_FLAGS="-lc++abi -Wl,--rpath=${LLVM_BUILD_LIBCXX}/lib -L${LLVM_BUILD_LIBCXX}/lib -fsanitize=memory -nostdinc++" + fi + + cd "${LLVM_BUILD}" + # Select components to build - CONFIG+=("-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi") + local ENABLED_LLVM_PROJECTS + ENABLED_LLVM_PROJECTS="clang" # cmake build if [[ "${enable_optimized}" == "1" && "${enable_debug}" != "1" ]]; then @@ -214,17 +163,46 @@ configure_llvm() { # Remove unneeded targets CONFIG+=( - "-DLLVM_INCLUDE_EXAMPLES=OFF" - "-DCLANG_INCLUDE_TESTS=OFF" - "-DLLVM_INCLUDE_BENCHMARKS=OFF" - "-DBUILD_SHARED_LIBS=ON" - "-DLLVM_OPTIMIZED_TABLEGEN=ON" - "-DLLVM_TARGETS_TO_BUILD=X86" + "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF" + "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF" + "-DLLVM_INCLUDE_TESTS:BOOL=OFF" + "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=TRUE" + "-DLLVM_TARGETS_TO_BUILD:STRING=X86" + "-DLLVM_INSTALL_UTILS:BOOL=TRUE" # Install FileCheck and Not + "-DLLVM_PARALLEL_LINK_JOBS=1" + "-DCLANG_INCLUDE_TESTS:BOOL=OFF" ) + if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then + # We have to build without libunwind if RTTI is disables + CONFIG+=("-DLLVM_ENABLE_PROJECTS=${ENABLED_LLVM_PROJECTS}") + else + CONFIG+=( + "-DLLVM_BUILD_LLVM_DYLIB:BOOL=ON" + "-DLLVM_LINK_LLVM_DYLIB:BOOL=ON" + "-DLLVM_BUILD_STATIC:BOOL=OFF" + "-DLIBCLANG_BUILD_STATIC:BOOL=OFF" + ) + CONFIG+=("-DLLVM_ENABLE_PROJECTS=${ENABLED_LLVM_PROJECTS};libcxx;libcxxabi;libunwind") + fi + if [[ -n ${SANITIZER_BUILD} ]]; then - CC="${SANITIZER_C_COMPILER}" - CXX="${SANITIZER_CXX_COMPILER}" + if [[ -n "${SANITIZER_C_COMPILER:-}" ]]; then + CC="${SANITIZER_C_COMPILER}" + fi + if [[ -n "${SANITIZER_CXX_COMPILER:-}" ]]; then + CXX="${SANITIZER_CXX_COMPILER}" + fi + + if [[ "${SANITIZER_BUILD}" == "address" ]] ; then + CONFIG+=("-DLLVM_USE_SANITIZER=Address") + fi + if [[ "${SANITIZER_BUILD}" == "undefined" ]] ; then + CONFIG+=("-DLLVM_USE_SANITIZER=Undefined") + fi + if [[ "${SANITIZER_BUILD}" == "memory" ]] ; then + CONFIG+=("-DLLVM_USE_SANITIZER=MemoryWithOrigins") + fi fi ( @@ -236,21 +214,19 @@ configure_llvm() { export "CXX=${CXX}" fi - if [[ -n "${LDFLAGS:-}" ]]; then - export "LDFLAGS=${LLVM_LDFLAGS}" + if [[ -n "${LLVM_LINKER_FLAGS:-}" ]]; then + export "LDFLAGS=${LLVM_LINKER_FLAGS}" fi - cmake "${CONFIG[@]}" "${LLVM_SRC_BASE}/llvm" + cmake -G Ninja "${CONFIG[@]}" "${LLVM_SRC_BASE}/llvm" ) } build_llvm() { - configure_llvm - ( - if [[ "${SANITIZER_BUILD}" == "memory" ]]; then - ninja clang|| return 1 - return 0 - fi + # Skip building if already finished + [[ -f "${LLVM_BUILD}/.build_finished" ]] && return 0 + configure_llvm || return 1 + ( if [[ -n "${CC:-}" ]]; then export "CC=${CC}" fi @@ -259,45 +235,23 @@ build_llvm() { export "CXX=${CXX}" fi - if [[ -n "${LDFLAGS:-}" ]]; then - export "LDFLAGS=${LLVM_LDFLAGS}" + if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then + ninja clang || return 1 + else + ninja || return 1 fi - - # Linking LLVM can require a lot of memory. - # First try multicore - if that doesn't work, try single core - (make "-j$(nproc)") || (make) || return 1 ) || return 1 touch "${LLVM_BUILD}/.build_finished" + } install_llvm() { - if [[ "${SANITIZER_BUILD}" != "memory" ]]; then - cd "${LLVM_BUILD}" - make "-j$(nproc)" install - cp "${LLVM_BUILD_BIN}/FileCheck" "${LLVM_INSTALL}/bin/" - cp "${LLVM_BUILD_BIN}/not" "${LLVM_INSTALL}/bin/" - - # Remove debug information from binaries - strip "${LLVM_INSTALL}/bin/"* || /bin/true - strip "${LLVM_INSTALL}/lib/libclang"* || /bin/true - strip "${LLVM_INSTALL}/lib/libLTO"* || /bin/true - else - # Handle memory sanitizer install - LLVM_PACKAGES=( - install-clang install-llvm-config install-llvm-objdump - install-llvm-link install-llvm-ar install-llvm-nm install-llvm-dis - install-clang-headers install-llvm-as - install-llvm-symbolizer install-LLVMSupport install-lli not FileCheck - install-llvm-headers - ) - - ninja "${LLVM_PACKAGES[@]}" - - for i in $(ninja -t targets | grep install-LLVM | cut -d : -f 1); do ninja "$i"; done - - cp "${LLVM_BUILD}/bin/FileCheck" "${LLVM_INSTALL}/bin/" - cp "${LLVM_BUILD}/bin/not" "${LLVM_INSTALL}/bin/" - fi + cd "${LLVM_BUILD}" + ninja install + # Remove debug information from binaries + strip "${LLVM_INSTALL}/bin/"* || /bin/true + strip "${LLVM_INSTALL}/lib/libclang"* || /bin/true + strip "${LLVM_INSTALL}/lib/libLTO"* || /bin/true touch "${LLVM_INSTALL}/.install_finished" } @@ -307,7 +261,7 @@ is_installed_llvm() { ( setup_build_variables_llvm # Check if the specific llvm-config exists - [[ -f "${LLVM_BUILD_BIN}/llvm-config" ]] + [[ -f "${LLVM_INSTALL}/bin/llvm-config" ]] [[ -f "${LLVM_INSTALL}/.install_finished" ]] ) || return 1 } @@ -327,9 +281,9 @@ setup_artifact_variables_llvm() { get_build_artifacts_llvm() { ( setup_build_variables_llvm + if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then + echo "${LLVM_BUILD_LIBCXX}" + fi echo "${LLVM_INSTALL}" - echo "${LLVM_SRC_BASE}" - [[ "${sanitizer}" == "memory" ]] && echo "${SANITIZER_LLVM_UNINSTRUMENTED}" - [[ "${sanitizer}" == "memory" ]] && echo "${SANITIZER_LLVM_LIBCXX}" ) } diff --git a/scripts/build/p-metasmt-linux-ubuntu.inc b/scripts/build/p-metasmt-linux-ubuntu.inc index 8bc03ceb..f0bc3f79 100644 --- a/scripts/build/p-metasmt-linux-ubuntu.inc +++ b/scripts/build/p-metasmt-linux-ubuntu.inc @@ -1,7 +1,3 @@ -setup_build_variables_metasmt() { - return 0 -} - install_build_dependencies_metasmt() { source "${DIR}/common-functions" with_sudo apt update -y @@ -21,10 +17,14 @@ install_build_dependencies_metasmt() { bison flex unzip - python autoconf python3 curl + python-is-python3 # Need for old Z3 + python3-distutils + python3-pip + python3-toml + default-jre # Needed for ANTLR ) #Install essential dependencies diff --git a/scripts/build/p-metasmt.inc b/scripts/build/p-metasmt.inc index 95a7f6a7..5aca84b4 100644 --- a/scripts/build/p-metasmt.inc +++ b/scripts/build/p-metasmt.inc @@ -13,6 +13,12 @@ download_metasmt() { git submodule update --init # Bootstrap git_clone_or_update https://github.com/agra-uni-bremen/dependencies.git "${METASMT_SRC_PATH}/dependencies" "development" + + # Apply existing patches if needed + if [ -f "${DIR}/patches/metasmt.patch" ]; then + cd "${METASMT_SRC_PATH}/dependencies" || (echo "Directory does not exist"; exit 1) + patch -p1 --follow-symlinks -i "${DIR}/patches/metasmt.patch" || return 1 + fi } build_metasmt() { @@ -24,8 +30,8 @@ build_metasmt() { cd "${METASMT_SRC_PATH}" ./bootstrap.sh -d deps -m RELEASE build -DmetaSMT_ENABLE_TESTS=off \ --build stp-git-basic --build boolector-git --build minisat-git \ - --build lingeling-bbc-9230380-161217 --build yices-2.6.1 --build Z3-4.8.4 \ - --build cvc4-1.6 \ + --build lingeling-bbc-9230380-161217 --build yices-2.6.2 --build Z3-4.8.6 \ + --build cvc4-1.8 \ --deps "${METASMT_DEPS_PATH}/" \ --install "${METASMT_INSTALL_PATH}/" \ -j "$(nproc)" \ @@ -51,7 +57,11 @@ is_installed_metasmt() { ( setup_build_variables_metasmt [[ -f "${METASMT_INSTALL_PATH}"/.is_installed ]] - ) + ) || return 1 +} + +setup_artifact_variables_metasmt() { + setup_build_variables_metasmt } get_docker_config_id_metasmt() { diff --git a/scripts/build/p-sanitizer-linux-ubuntu.inc b/scripts/build/p-sanitizer-linux-ubuntu.inc index 30e153bb..5aa6480a 100644 --- a/scripts/build/p-sanitizer-linux-ubuntu.inc +++ b/scripts/build/p-sanitizer-linux-ubuntu.inc @@ -1,5 +1,5 @@ install_binary_artifact_sanitizer() { - if [[ -z "${SANITIZER_BUILD:=}" ]]; then + if [[ -z "${SANITIZER_BUILD:-}" ]]; then return 1 fi @@ -7,34 +7,46 @@ install_binary_artifact_sanitizer() { local SANITIZER_LLVM_VERSION_MAJOR="${SANITIZER_LLVM_VERSION/.*/}" local version="-${SANITIZER_LLVM_VERSION_MAJOR}" - # Add certificate - with_sudo apt update -y - dependencies=( - ca-certificates - wget - lsb-release - gnupg - ) - - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | with_sudo apt-key add - + local LLVM_UPSTREAM_USED=0 + # Check if package in standard repository otherwise use upstream + with_sudo apt-get update -y + if ! apt-cache show "llvm${version}"; then + if [[ -z "$(which wget)" ]]; then + # Add certificate + with_sudo apt-get update -y + dependencies=( + ca-certificates + wget + lsb-release + gnupg + ) + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" + fi - # Add repository - codename="$(lsb_release --codename --short)" - apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main" - if [[ ! $(grep -rq "${apt_entry}" /etc/apt) ]]; then - echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list - with_sudo apt update -y + # Add LLVM upstream repository if available + codename="$(lsb_release --codename --short)" + if wget -q "https://apt.llvm.org/${codename}/dists/llvm-toolchain-${codename}${version}/"; then + LLVM_UPSTREAM_USED=1 + apt_entry="deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}${version} main" + if ! grep -rq "${apt_entry}" /etc/apt; then + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| with_sudo apt-key add - + echo "${apt_entry}" | with_sudo tee -a /etc/apt/sources.list + with_sudo apt-get update -y + fi + fi fi - with_sudo apt update -y dependencies=( "llvm${version}" "clang${version}" ) + if [[ "${SANITIZER_LLVM_VERSION_MAJOR}" -ge 14 && "${LLVM_UPSTREAM_USED}" -eq 1 ]]; then + dependencies+=("libclang-rt${version}-dev") + fi + #Install essential dependencies - with_sudo apt -y --no-install-recommends install "${dependencies[@]}" || return 1 + with_sudo apt-get -y --no-install-recommends install "${dependencies[@]}" || return 1 } get_docker_config_id_sanitizer() { diff --git a/scripts/build/p-sanitizer-linux.inc b/scripts/build/p-sanitizer-linux.inc index fa9f35f3..6cfd5920 100644 --- a/scripts/build/p-sanitizer-linux.inc +++ b/scripts/build/p-sanitizer-linux.inc @@ -15,23 +15,23 @@ setup_artifact_variables_sanitizer() { SANITIZER_BITCODE_CC="${bin_path}/clang" SANITIZER_BITCODE_CXX="${bin_path}/clang++" - if [[ -z ${SANITIZER_C_COMPILER:=} ]]; then + if [[ -z "${SANITIZER_C_COMPILER:-}" ]]; then SANITIZER_C_COMPILER="${SANITIZER_BITCODE_CC}" fi - if [[ -z ${SANITIZER_CXX_COMPILER:=} ]]; then + if [[ -z "${SANITIZER_CXX_COMPILER:-}" ]]; then SANITIZER_CXX_COMPILER="${SANITIZER_BITCODE_CXX}" fi } # Check if the binary artifact is installed is_installed_sanitizer() { - if [[ -z ${SANITIZER_BUILD:-} ]]; then + if [[ -z "${SANITIZER_BUILD:-}" ]]; then return 1 fi local SANITIZER_LLVM_VERSION_MAJOR="${SANITIZER_LLVM_VERSION/.*/}" local SANITIZER_LLVM_VERSION_MINOR="${SANITIZER_LLVM_VERSION/*./}" - local version=="${SANITIZER_LLVM_VERSION_MAJOR}" + local version="${SANITIZER_LLVM_VERSION_MAJOR}" # Check if clang with the right version exists which "clang-${version}" diff --git a/scripts/build/p-sqlite-linux-ubuntu.inc b/scripts/build/p-sqlite-linux-ubuntu.inc index e0f317cb..e23da9ad 100644 --- a/scripts/build/p-sqlite-linux-ubuntu.inc +++ b/scripts/build/p-sqlite-linux-ubuntu.inc @@ -13,6 +13,7 @@ install_build_dependencies_sqlite() { } install_binary_artifact_sqlite() { [[ "${SANITIZER_SUFFIX}x" == "x" ]] || return 1 + [[ "${SQLITE_VERSION}" == "322" ]] || return 1 source "${DIR}/common-functions" with_sudo apt update -y @@ -24,14 +25,7 @@ install_binary_artifact_sqlite() { # Check if the binary artifact is installed is_installed_sqlite() { + [[ -z "${SANITIZER_SUFFIX:-}" ]] || return 1 [[ -f /usr/lib/x86_64-linux-gnu/libsqlite3.so ]] || return 1 [[ -f /usr/include/sqlite3.h ]] || return 1 -} - -get_docker_config_id_sqlite() { - return 0 -} - -get_build_artifacts_sqlite() { - return 0 } \ No newline at end of file diff --git a/scripts/build/p-sqlite.inc b/scripts/build/p-sqlite.inc index 3c41ccc6..1182e590 100644 --- a/scripts/build/p-sqlite.inc +++ b/scripts/build/p-sqlite.inc @@ -1,8 +1,7 @@ setup_build_variables_sqlite() { - SQLITE_VERSION="3370200" SQLITE_SRC_PATH="${BASE}/sqlite-amalgamation-${SQLITE_VERSION}" - SQLITE_INSTALL_PATH=${SQLITE_SRC_PATH} SQLITE_SUFFIX="${SANITIZER_SUFFIX}" + SQLITE_INSTALL_PATH="${SQLITE_SRC_PATH}${SANITIZER_SUFFIX}" } download_sqlite() { @@ -20,11 +19,14 @@ build_sqlite() { CFLAGS=("${SANITIZER_C_FLAGS[@]}") COMP="${SANITIZER_C_COMPILER}" else - COMP="${BITCODE_CC}" - [[ -z "${COMP}" ]] && COMP="$(which gcc)" + [[ -z "${COMP:-}" ]] && COMP="$(which gcc)" fi - "${COMP}" "${CFLAGS[@]}" -fPIC -o libsqlite3.so -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -shared sqlite3.c + mkdir -p "${SQLITE_INSTALL_PATH}" + "${COMP}" "${CFLAGS[@]}" -fPIC -o "${SQLITE_INSTALL_PATH}"/libsqlite3.so -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -shared sqlite3.c + if [[ -n "${SANITIZER_SUFFIX:-}" ]]; then + cp "${SQLITE_SRC_PATH}/"sqlite3.h "${SQLITE_INSTALL_PATH}/" + fi } install_sqlite() { @@ -45,7 +47,7 @@ setup_artifact_variables_sqlite() { get_docker_config_id_sqlite() { ( setup_build_variables_sqlite - echo "${SQLITE_SUFFIX}" + echo "${SQLITE_VERSION}${SQLITE_SUFFIX}" ) } @@ -53,5 +55,6 @@ get_build_artifacts_sqlite() { ( setup_build_variables_sqlite echo "${SQLITE_SRC_PATH}" + echo "${SQLITE_INSTALL_PATH}" ) } diff --git a/scripts/build/p-uclibc-linux-ubuntu.inc b/scripts/build/p-uclibc-linux-ubuntu.inc index e10304aa..936593cc 100644 --- a/scripts/build/p-uclibc-linux-ubuntu.inc +++ b/scripts/build/p-uclibc-linux-ubuntu.inc @@ -7,7 +7,7 @@ install_build_dependencies_uclibc() { build-essential ca-certificates git - python + python3 libncurses5-dev wget ) diff --git a/scripts/build/p-z3-linux-ubuntu.inc b/scripts/build/p-z3-linux-ubuntu.inc index 41130232..9e3b7b7e 100644 --- a/scripts/build/p-z3-linux-ubuntu.inc +++ b/scripts/build/p-z3-linux-ubuntu.inc @@ -5,7 +5,7 @@ install_build_dependencies_z3() { dependencies=( build-essential - python + python3 git wget ca-certificates diff --git a/scripts/build/p-z3.inc b/scripts/build/p-z3.inc index f6db455b..80506794 100644 --- a/scripts/build/p-z3.inc +++ b/scripts/build/p-z3.inc @@ -31,7 +31,7 @@ build_z3() { LDFLAGS="${LDFLAGS}" \ CC="${CC}" \ CXX="${CXX}" \ - python scripts/mk_make.py --prefix "${Z3_INSTALL_PATH}" -b "build${Z3_SUFFIX}" + python3 scripts/mk_make.py --prefix "${Z3_INSTALL_PATH}" -b "build${Z3_SUFFIX}" cd "build${Z3_SUFFIX}" make -j$(nproc) || make } diff --git a/scripts/build/patches/libcxx110.patch b/scripts/build/patches/libcxx110.patch new file mode 100644 index 00000000..03e7dbb7 --- /dev/null +++ b/scripts/build/patches/libcxx110.patch @@ -0,0 +1,13 @@ +--- a/libcxx/include/__config ++++ b/libcxx/include/__config +@@ -1167,10 +1167,6 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( + _LIBCPP_HAS_NO_THREADS is defined. + #endif + +-#if defined(__STDCPP_THREADS__) && defined(_LIBCPP_HAS_NO_THREADS) +-#error _LIBCPP_HAS_NO_THREADS cannot be set when __STDCPP_THREADS__ is set. +-#endif +- + #if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(__STDCPP_THREADS__) + #define __STDCPP_THREADS__ 1 + #endif \ No newline at end of file diff --git a/scripts/build/patches/llvm110.patch b/scripts/build/patches/llvm110.patch index a764f9bc..83014b52 100644 --- a/scripts/build/patches/llvm110.patch +++ b/scripts/build/patches/llvm110.patch @@ -49,4 +49,74 @@ - _(CYSETTIMEOUT, NONE, 0); _(EQL_EMANCIPATE, WRITE, struct_ifreq_sz); _(EQL_ENSLAVE, WRITE, struct_ifreq_sz); - _(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz); \ No newline at end of file + _(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz); +--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp ++++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp +@@ -169,7 +169,11 @@ bool SupportsColoredOutput(fd_t fd) { + + #if !SANITIZER_GO + // TODO(glider): different tools may require different altstack size. +-static const uptr kAltStackSize = SIGSTKSZ * 4; // SIGSTKSZ is not enough. ++static uptr GetAltStackSize() { ++ // SIGSTKSZ is not enough. ++ static const uptr kAltStackSize = SIGSTKSZ * 4; ++ return kAltStackSize; ++} + + void SetAlternateSignalStack() { + stack_t altstack, oldstack; +@@ -180,10 +184,10 @@ void SetAlternateSignalStack() { + // TODO(glider): the mapped stack should have the MAP_STACK flag in the + // future. It is not required by man 2 sigaltstack now (they're using + // malloc()). +- void* base = MmapOrDie(kAltStackSize, __func__); ++ void *base = MmapOrDie(GetAltStackSize(), __func__); + altstack.ss_sp = (char*) base; + altstack.ss_flags = 0; +- altstack.ss_size = kAltStackSize; ++ altstack.ss_size = GetAltStackSize(); + CHECK_EQ(0, sigaltstack(&altstack, nullptr)); + } + +@@ -191,7 +195,7 @@ void UnsetAlternateSignalStack() { + stack_t altstack, oldstack; + altstack.ss_sp = nullptr; + altstack.ss_flags = SS_DISABLE; +- altstack.ss_size = kAltStackSize; // Some sane value required on Darwin. ++ altstack.ss_size = GetAltStackSize(); // Some sane value required on Darwin. + CHECK_EQ(0, sigaltstack(&altstack, &oldstack)); + UnmapOrDie(oldstack.ss_sp, oldstack.ss_size); + } +--- a/llvm/lib/Transforms/CMakeLists.txt ++++ b/llvm/lib/Transforms/CMakeLists.txt +@@ -5,7 +5,6 @@ add_subdirectory(InstCombine) + add_subdirectory(Scalar) + add_subdirectory(IPO) + add_subdirectory(Vectorize) +-add_subdirectory(Hello) + add_subdirectory(ObjCARC) + add_subdirectory(Coroutines) + add_subdirectory(CFGuard) +--- a/llvm/test/CMakeLists.txt ++++ b/llvm/test/CMakeLists.txt +@@ -47,7 +47,6 @@ configure_lit_site_cfg( + set(LLVM_TEST_DEPENDS + BugpointPasses + FileCheck +- LLVMHello + UnitTests + bugpoint + count +--- a/libcxx/include/__config ++++ b/libcxx/include/__config +@@ -1167,10 +1167,6 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( + _LIBCPP_HAS_NO_THREADS is defined. + #endif + +-#if defined(__STDCPP_THREADS__) && defined(_LIBCPP_HAS_NO_THREADS) +-#error _LIBCPP_HAS_NO_THREADS cannot be set when __STDCPP_THREADS__ is set. +-#endif +- + #if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(__STDCPP_THREADS__) + #define __STDCPP_THREADS__ 1 + #endif \ No newline at end of file diff --git a/scripts/build/patches/llvm120.patch b/scripts/build/patches/llvm120.patch new file mode 100644 index 00000000..f72eb38d --- /dev/null +++ b/scripts/build/patches/llvm120.patch @@ -0,0 +1,8 @@ +diff --git a/msan_suppressions.txt b/msan_suppressions.txt +new file mode 100644 +index 000000000..f8c825e6f +--- /dev/null ++++ b/msan_suppressions.txt +@@ -0,0 +1,2 @@ ++# Ignore llvm-config issue ++mainfile:llvm-config.cpp diff --git a/scripts/build/patches/llvm90.patch b/scripts/build/patches/llvm90.patch index e99827b6..3ff0dc29 100644 --- a/scripts/build/patches/llvm90.patch +++ b/scripts/build/patches/llvm90.patch @@ -1,18 +1,28 @@ ---- a/llvm/tools/llvm-shlib/CMakeLists.txt -+++ b/llvm/tools/llvm-shlib/CMakeLists.txt -@@ -41,7 +41,9 @@ if(LLVM_BUILD_LLVM_DYLIB) - - add_llvm_library(LLVM SHARED DISABLE_LLVM_LINK_LLVM_DYLIB SONAME ${SOURCES}) +diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +index 490a04b21..42e43a044 100644 +--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc ++++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +@@ -366,15 +366,6 @@ static void ioctl_table_fill() { -+ if(LIB_NAMES) - list(REMOVE_DUPLICATES LIB_NAMES) -+ endif() - if(("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") OR (MINGW) OR (HAIKU) - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "GNU") ---- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp -+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp -@@ -143,7 +143,6 @@ typedef struct user_fpregs elf_fpregset_t; + #if SANITIZER_LINUX && !SANITIZER_ANDROID + // _(SIOCDEVPLIP, WRITE, struct_ifreq_sz); // the same as EQL_ENSLAVE +- _(CYGETDEFTHRESH, WRITE, sizeof(int)); +- _(CYGETDEFTIMEOUT, WRITE, sizeof(int)); +- _(CYGETMON, WRITE, struct_cyclades_monitor_sz); +- _(CYGETTHRESH, WRITE, sizeof(int)); +- _(CYGETTIMEOUT, WRITE, sizeof(int)); +- _(CYSETDEFTHRESH, NONE, 0); +- _(CYSETDEFTIMEOUT, NONE, 0); +- _(CYSETTHRESH, NONE, 0); +- _(CYSETTIMEOUT, NONE, 0); + _(EQL_EMANCIPATE, WRITE, struct_ifreq_sz); + _(EQL_ENSLAVE, WRITE, struct_ifreq_sz); + _(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz); +diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +index b7fa6e8f7..fa981e129 100644 +--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc ++++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +@@ -126,7 +126,6 @@ typedef struct user_fpregs elf_fpregset_t; # include <sys/procfs.h> #endif #include <sys/user.h> @@ -20,15 +30,15 @@ #include <linux/if_eql.h> #include <linux/if_plip.h> #include <linux/lp.h> -@@ -459,7 +458,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); +@@ -437,7 +436,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); - #if SANITIZER_GLIBC + #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct); - unsigned struct_cyclades_monitor_sz = sizeof(struct cyclades_monitor); #if EV_VERSION > (0x010000) unsigned struct_input_keymap_entry_sz = sizeof(struct input_keymap_entry); #else -@@ -823,15 +821,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); +@@ -803,15 +801,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); #endif // SANITIZER_LINUX #if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -44,21 +54,61 @@ unsigned IOCTL_EQL_EMANCIPATE = EQL_EMANCIPATE; unsigned IOCTL_EQL_ENSLAVE = EQL_ENSLAVE; unsigned IOCTL_EQL_GETMASTRCFG = EQL_GETMASTRCFG; ---- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc -+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc -@@ -370,15 +370,6 @@ static void ioctl_table_fill() { +@@ -1126,8 +1115,9 @@ CHECK_SIZE_AND_OFFSET(ipc_perm, uid); + CHECK_SIZE_AND_OFFSET(ipc_perm, gid); + CHECK_SIZE_AND_OFFSET(ipc_perm, cuid); + CHECK_SIZE_AND_OFFSET(ipc_perm, cgid); +-#if !defined(__aarch64__) || !SANITIZER_LINUX || __GLIBC_PREREQ (2, 21) +-/* On aarch64 glibc 2.20 and earlier provided incorrect mode field. */ ++#if !SANITIZER_LINUX || __GLIBC_PREREQ (2, 31) ++/* glibc 2.30 and earlier provided 16-bit mode field instead of 32-bit ++ on many architectures. */ + CHECK_SIZE_AND_OFFSET(ipc_perm, mode); + #endif - #if SANITIZER_GLIBC - // _(SIOCDEVPLIP, WRITE, struct_ifreq_sz); // the same as EQL_ENSLAVE -- _(CYGETDEFTHRESH, WRITE, sizeof(int)); -- _(CYGETDEFTIMEOUT, WRITE, sizeof(int)); -- _(CYGETMON, WRITE, struct_cyclades_monitor_sz); -- _(CYGETTHRESH, WRITE, sizeof(int)); -- _(CYGETTIMEOUT, WRITE, sizeof(int)); -- _(CYSETDEFTHRESH, NONE, 0); -- _(CYSETDEFTIMEOUT, NONE, 0); -- _(CYSETTHRESH, NONE, 0); -- _(CYSETTIMEOUT, NONE, 0); - _(EQL_EMANCIPATE, WRITE, struct_ifreq_sz); - _(EQL_ENSLAVE, WRITE, struct_ifreq_sz); - _(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz); \ No newline at end of file +diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +index f1a4fd7d3..029a209fc 100644 +--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h ++++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +@@ -203,26 +203,13 @@ namespace __sanitizer { + u64 __unused1; + u64 __unused2; + #elif defined(__sparc__) +-#if defined(__arch64__) + unsigned mode; +- unsigned short __pad1; +-#else +- unsigned short __pad1; +- unsigned short mode; + unsigned short __pad2; +-#endif + unsigned short __seq; + unsigned long long __unused1; + unsigned long long __unused2; +-#elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__) +- unsigned int mode; +- unsigned short __seq; +- unsigned short __pad1; +- unsigned long __unused1; +- unsigned long __unused2; + #else +- unsigned short mode; +- unsigned short __pad1; ++ unsigned int mode; + unsigned short __seq; + unsigned short __pad2; + #if defined(__x86_64__) && !defined(_LP64) +diff --git a/llvm/tools/llvm-shlib/CMakeLists.txt b/llvm/tools/llvm-shlib/CMakeLists.txt +index 901f55c99..7cb78afe5 100644 +--- a/llvm/tools/llvm-shlib/CMakeLists.txt ++++ b/llvm/tools/llvm-shlib/CMakeLists.txt +@@ -44,7 +44,9 @@ if(LLVM_BUILD_LLVM_DYLIB) + endif() + add_llvm_library(LLVM SHARED DISABLE_LLVM_LINK_LLVM_DYLIB SONAME ${INSTALL_WITH_TOOLCHAIN} ${SOURCES}) + ++ if(LIB_NAMES) + list(REMOVE_DUPLICATES LIB_NAMES) ++ endif() + if(("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") OR (MINGW) OR (HAIKU) + OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "GNU") diff --git a/scripts/build/patches/metasmt.patch b/scripts/build/patches/metasmt.patch new file mode 100644 index 00000000..f14b41b9 --- /dev/null +++ b/scripts/build/patches/metasmt.patch @@ -0,0 +1,72 @@ +--- /dev/null ++++ b/Z3-4.8.6/setup.sh +@@ -0,0 +1,8 @@ ++#!/bin/sh ++ ++version=4.8.6 ++branch=z3-4.8.6 ++ ++cmake_files_dir=$base_dir/Z3-git ++ ++source $base_dir/Z3-git/shared.sh +--- a/cvc4-1.5/shared.sh ++++ b/cvc4-1.5/shared.sh +@@ -9,10 +9,10 @@ if [ -z "$package_dir" ] ; then + exit 1 + fi + +-package=cvc4 ++package=CVC4 + source="$package-$version.tar.gz" + build_dir=$build/$package-$version +-url="http://cvc4.cs.stanford.edu/downloads/builds/src/$source" ++url="https://github.com/CVC4/CVC4-archived/archive/refs/tags/$version.tar.gz" + + if [ -z "$BOOST_ROOT" ]; then + dependencies="$DEPS_BOOST" +@@ -24,7 +24,7 @@ fi + unpack() { + cd $cache && + tar -xf $source && +- mv -f $package-$version $build_dir ++ mv -f $package-archived-$version $build_dir + cd $build_dir + } + +@@ -39,7 +39,8 @@ build_install() { + contrib/get-antlr-3.4 + # build CVC4 with --bsd to allow usage under the terms of + # the modified BSD license. +- ./configure --prefix="$target" --bsd --with-antlr-dir=$build_dir/antlr-3.4 --with-boost=$boost_path ANTLR=$build_dir/antlr-3.4/bin/antlr3 && ++ ./configure.sh --prefix="$target" --antlr-dir=$build_dir/antlr-3.4 && ++ cd build && + make -j $num_threads && + make install && + cp -f "$package_dir/CVC4Config.cmake" "$target/CVC4Config.cmake" +--- /dev/null ++++ b/cvc4-1.8/CVC4Config.cmake +@@ -0,0 +1,5 @@ ++get_filename_component(CVC4_DIR ${CMAKE_CURRENT_LIST_FILE} PATH) ++set(CVC4_BIN_DIRS ${CVC4_DIR}/bin ) ++set(CVC4_INCLUDE_DIRS ${CVC4_DIR}/include ) ++set(CVC4_INCLUDE_DIR ${CVC4_INCLUDE_DIRS} ) ++set(CVC4_LIBRARIES ${CVC4_DIR}/lib/libcvc4.so ) +--- /dev/null ++++ b/cvc4-1.8/setup.sh +@@ -0,0 +1,5 @@ ++#!/bin/sh ++ ++version="1.8" ++source $base_dir/cvc4-1.5/shared.sh ++ +--- /dev/null ++++ b/yices-2.6.2/setup.sh +@@ -0,0 +1,8 @@ ++#!/bin/sh ++ ++version=2.6.2 ++branch=Yices-2.6.2 ++ ++config_files_dir=$base_dir/yices-git ++ ++source $base_dir/yices-git/shared.sh diff --git a/scripts/build/run-tests.sh b/scripts/build/run-tests.sh index 7eb8b4fa..4b9b972e 100755 --- a/scripts/build/run-tests.sh +++ b/scripts/build/run-tests.sh @@ -38,7 +38,7 @@ run_tests() { cd "${build_dir}" # Remove klee from PATH - export PATH=${PATH%":/home/klee/klee_build/bin"} + export PATH=${PATH/":/home/klee/klee_build/bin"/} if which klee; then return 1 # should not happen fi diff --git a/scripts/build/v-klee.inc b/scripts/build/v-klee.inc index 78c4e1d1..4b0f1085 100644 --- a/scripts/build/v-klee.inc +++ b/scripts/build/v-klee.inc @@ -21,8 +21,8 @@ artifact_dependency_klee(){ dependencies+=(uclibc) fi - [[ "${USE_TCMALLOC}" -eq 1 ]] && dependencies+=("tcmalloc") - [[ "${USE_LIBCXX}" -eq 1 ]] && dependencies+=("libcxx") + [[ "${USE_TCMALLOC:-}" -eq 1 ]] && dependencies+=("tcmalloc") + [[ "${USE_LIBCXX:-}" -eq 1 ]] && dependencies+=("libcxx") for d in "${dependencies[@]}"; do echo "${d}" @@ -36,8 +36,8 @@ runtime_artifact_dependency_klee(){ fi dependencies+=("llvm" "solvers" "gtest" "sqlite") - [[ "${USE_TCMALLOC}" -eq 1 ]] && dependencies+=("tcmalloc") - [[ "${USE_LIBCXX}" -eq 1 ]] && dependencies+=("libcxx") + [[ "${USE_TCMALLOC:-}" -eq 1 ]] && dependencies+=("tcmalloc") + [[ "${USE_LIBCXX:-}" -eq 1 ]] && dependencies+=("libcxx") for d in "${dependencies[@]}"; do echo "${d}" diff --git a/scripts/build/v-libcxx.inc b/scripts/build/v-libcxx.inc index 4a60d69e..5f157701 100644 --- a/scripts/build/v-libcxx.inc +++ b/scripts/build/v-libcxx.inc @@ -7,6 +7,6 @@ required_variables_check_libcxx() { } artifact_dependency_libcxx(){ - echo "clang" + echo "llvm" echo "cmake" } diff --git a/scripts/build/v-sanitizer.inc b/scripts/build/v-sanitizer.inc index b86ef23f..09005df0 100644 --- a/scripts/build/v-sanitizer.inc +++ b/scripts/build/v-sanitizer.inc @@ -75,8 +75,8 @@ setup_variables_sanitizer() { # Undefined Behaviour Sanitizer if [ "${sanitizer}" == "undefined" ]; then echo "Using UBSan" - flags=("-fsanitize=undefined" "-fno-omit-frame-pointer" "-g" - "-fno-sanitize-recover=undefined") + # Allow to recover from any undefined behaviour + flags=("-fsanitize=undefined" "-fno-omit-frame-pointer" "-g" "-fsanitize-recover=signed-integer-overflow,alignment,vptr") SANITIZER_CXX_FLAGS+=("${flags[@]}") SANITIZER_C_FLAGS+=("${flags[@]}") SANITIZER_LD_FLAGS+=("${flags[@]}") @@ -87,6 +87,30 @@ setup_variables_sanitizer() { # Memory Sanitizer if [ "${sanitizer}" == "memory" ]; then echo "Using Memory Sanitizer" + # Generate full sanitizer ignore file. + cat "${DIR}/sanitizer/"*.txt > "${BASE}/sanitizer_ignore.txt" + flags=("-fsanitize=memory" "-fno-omit-frame-pointer" "-g" "-fsanitize-memory-track-origins" "-fsanitize-ignorelist=${BASE}/sanitizer_ignore.txt") + + if [[ -n "${LLVM_VERSION:-}" ]]; then + # Workaround as we have to pull in MemSan-instrumented libcxx + local LLVM_BUILD_LIBCXX="${BASE}/llvm-${LLVM_VERSION}0-install_NO_D_A_memsan-libcxx" + if [[ -d "${LLVM_BUILD_LIBCXX}" ]]; then + SANITIZER_CXX_FLAGS+=( + "-nostdinc++" + "-isystem ${LLVM_BUILD_LIBCXX}/include" + "-isystem ${LLVM_BUILD_LIBCXX}/include/c++/v1" + ) + SANITIZER_LD_FLAGS+=( + "-lc++abi" + "-Wl,--rpath=${LLVM_BUILD_LIBCXX}/lib" + "-L${LLVM_BUILD_LIBCXX}/lib" + "-stdlib=libc++" + ) + fi + fi + SANITIZER_CXX_FLAGS+=("${flags[@]}") + SANITIZER_C_FLAGS+=("${flags[@]}") + SANITIZER_LD_FLAGS+=("${flags[@]}") SANITIZER_SUFFIX+="_memsan" continue fi diff --git a/scripts/build/v-sqlite.inc b/scripts/build/v-sqlite.inc index 366f442f..17f248cc 100644 --- a/scripts/build/v-sqlite.inc +++ b/scripts/build/v-sqlite.inc @@ -1,3 +1,5 @@ artifact_dependency_sqlite=("sanitizer") -required_variables_sqlite=("") +required_variables_sqlite=( + "SQLITE_VERSION" +) diff --git a/scripts/build/v-stp.inc b/scripts/build/v-stp.inc index 0885a3d1..17a84acc 100644 --- a/scripts/build/v-stp.inc +++ b/scripts/build/v-stp.inc @@ -5,4 +5,11 @@ required_variables_stp=( ) # On which artifacts does STP depend on -artifact_dependency_stp=("sanitizer") \ No newline at end of file +artifact_dependency_stp=("sanitizer") + +artifact_dependency_stp() { + echo "sanitizer" + if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then + echo "llvm" + fi +} \ No newline at end of file diff --git a/scripts/build/v-uclibc.inc b/scripts/build/v-uclibc.inc index 7cdedfcd..3c1c3d73 100644 --- a/scripts/build/v-uclibc.inc +++ b/scripts/build/v-uclibc.inc @@ -6,5 +6,5 @@ required_variables_uclibc=( artifact_dependency_uclibc(){ # Add llvm if needed; otherwise only use clang package # TODO this is quite distribution specific; should be handled in a more general case - echo "clang" + echo "llvm" } |