Update build scripts

* Support for Ubuntu 22.04
* Remove support for Python2
* Better separation between sanitizer builds and non-sanitizer builds
* Fix build of metaSMT on newer Ubuntu versions
* Use ninja to build LLVM
* Simplifying building arbitrary LLVM configurations, e.g. different
  LLVM sanitizer builds (MemSan, UBSan, ASan)
* Use MemSan with origin tracking
* Build sqlite3 container correctly
* Add support to provide sqlite3 version number

1 files changed, 93 insertions, 139 deletions

diff --git a/scripts/build/p-llvm.inc b/scripts/build/p-llvm.inc
index d1b9374e..abf895ae 100644
--- a/scripts/build/p-llvm.inc
+++ b/scripts/build/p-llvm.inc
@@ -55,38 +55,14 @@ setup_build_variables_llvm() {
   LLVM_BIN="${LLVM_INSTALL}/bin"
   LLVM_BUILD_BIN="${LLVM_BUILD}/bin"
 
+  if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then
+    LLVM_BUILD_LIBCXX="${LLVM_INSTALL}-libcxx"
+  fi
+
+
   LLVM_CONFIG="${LLVM_BIN}/llvm-config"
   BITCODE_CC="${LLVM_BIN}/clang"
   BITCODE_CXX="${LLVM_BIN}/clang++"
-
-  for sanitizer in "${SANITIZERS[@]}"; do
-    [[ -z "${sanitizer}" ]] && continue
-    # Undefined Behaviour Sanitizer
-    if [ "${sanitizer}" == "memory" ]; then
-      SANITIZER_LLVM_UNINSTRUMENTED="${LLVM_BUILD}_uninstrumented"
-      SANITIZER_LLVM_LIBCXX="${LLVM_BUILD}_libcxx"
-
-      MSAN_LINK_FLAGS=("-lc++abi" "-Wl,--rpath=${SANITIZER_LLVM_LIBCXX}/lib" "-L${SANITIZER_LLVM_LIBCXX}/lib")
-      MSAN_FLAGS=("${MSAN_LINK_FLAGS[@]}" -nostdinc++
-        -isystem "${SANITIZER_LLVM_LIBCXX}/include"
-        -isystem "${SANITIZER_LLVM_LIBCXX}/include/c++/v1"
-        "-fsanitize=memory"
-        "-fsanitize-memory-track-origins"
-        -w -fno-omit-frame-pointer -g)
-      SANITIZER_CXX_FLAGS+=("${MSAN_FLAGS[@]}" "-stdlib=libc++")
-      SANITIZER_C_FLAGS+=("${MSAN_FLAGS[@]}")
-      SANITIZER_LD_FLAGS+=("${MSAN_LINK_FLAGS[@]}")
-      SANITIZER_C_COMPILER="${SANITIZER_LLVM_UNINSTRUMENTED}/bin/clang"
-      SANITIZER_CXX_COMPILER="${SANITIZER_LLVM_UNINSTRUMENTED}/bin/clang++"
-
-      # Use the uninstrumented compiler
-      BITCODE_CC="${SANITIZER_C_COMPILER}"
-      BITCODE_CXX="${SANITIZER_CXX_COMPILER}"
-      # But point to the instrumented llvm-config
-      LLVM_CONFIG="${LLVM_BIN}/llvm-config"
-      continue
-    fi
-  done
 }
 
 download_llvm() {
@@ -117,71 +93,44 @@ configure_llvm() {
   local disable_assertions=$(to_bool "${DISABLE_ASSERTIONS}")
   local requires_rtti=$(to_bool "${REQUIRES_RTTI}")
 
-  # For memory sanitizer, we have a multi-stage build process
-  if [[ "${SANITIZER_BUILD}" == "memory" ]]; then
-    # Build uninstrumented compiler
-    mkdir -p "${SANITIZER_LLVM_UNINSTRUMENTED}"
-    cd "${SANITIZER_LLVM_UNINSTRUMENTED}"
-    cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
-      "-DLLVM_OPTIMIZED_TABLEGEN=ON" \
-      "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi" \
-      "-DLLVM_TARGETS_TO_BUILD=X86" \
-      "${LLVM_SRC_BASE}/llvm"
-    ninja compiler-rt cxx cxxabi clang || return 1
-
-    # Build instrumented libc/libc++
-    mkdir -p "${SANITIZER_LLVM_LIBCXX}"
-    cd "${SANITIZER_LLVM_LIBCXX}"
-    cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-      "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi" \
-      -DLLVM_USE_SANITIZER=MemoryWithOrigins \
-      "-DCMAKE_C_COMPILER=${SANITIZER_C_COMPILER}" \
-      "-DCMAKE_CXX_COMPILER=${SANITIZER_CXX_COMPILER}" \
-      "-DLLVM_OPTIMIZED_TABLEGEN=ON" \
-      "-DLLVM_TARGETS_TO_BUILD=X86" \
-      "${LLVM_SRC_BASE}/llvm"
-    ninja compiler-rt cxx cxxabi || return 1
-
-    # Build instrumented clang
-    mkdir -p "${LLVM_BUILD}"
-    cd "${LLVM_BUILD}"
-    C_F="${SANITIZER_C_FLAGS[*]}"
-    CXX_F="${SANITIZER_CXX_FLAGS[*]}"
-    LD_F="${SANITIZER_LD_FLAGS[*]}"
-    cmake -GNinja \
-      "-DCMAKE_C_COMPILER=${SANITIZER_C_COMPILER}" \
-      "-DCMAKE_CXX_COMPILER=${SANITIZER_CXX_COMPILER}" \
-      -DCMAKE_C_FLAGS="$C_F" \
-      -DCMAKE_CXX_FLAGS="${CXX_F}" \
-      -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-      "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi" \
-      -DLLVM_ENABLE_ASSERTIONS=On \
-      -DLLVM_USE_SANITIZER=MemoryWithOrigins \
-      -DLLVM_ENABLE_LIBCXX=ON \
-      -DCMAKE_EXE_LINKER_FLAGS="${LD_F}" \
-      -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL}" \
-      -DBUILD_SHARED_LIBS=ON \
-      -DLLVM_OPTIMIZED_TABLEGEN=ON \
-      -DLLVM_TARGETS_TO_BUILD=X86 \
-      "${LLVM_SRC_BASE}/llvm"
-    return 0
-  fi
-
   # Configure; build; and install
   mkdir -p "${LLVM_BUILD}"
-  cd "${LLVM_BUILD}"
-
-  # Skip building if already finished
-  [[ -e "${LLVM_BUILD}/.build_finished" ]] && return 0
 
   # Configure LLVM
   CONFIG=(
     "-DCMAKE_INSTALL_PREFIX=${LLVM_INSTALL}"
-    "-DLLVM_BUILD_LLVM_DYLIB=TRUE"
   )
 
+
+  if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then
+    # Two stage process, build libcxx and libcxxabi
+
+    # Skip building if already finished
+    if [[ ! -f "${LLVM_BUILD_LIBCXX}/.build_finished" ]]; then
+      mkdir -p "${LLVM_BUILD_LIBCXX}"
+      cd "${LLVM_BUILD_LIBCXX}"
+      cmake -GNinja "${LLVM_SRC_BASE}/llvm" \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \
+        -DCMAKE_C_COMPILER="${SANITIZER_C_COMPILER}" \
+        -DCMAKE_CXX_COMPILER="${SANITIZER_CXX_COMPILER}" \
+        -DLLVM_USE_SANITIZER=MemoryWithOrigins
+      cmake --build . -- cxx cxxabi || return 1
+      touch "${LLVM_BUILD_LIBCXX}/.build_finished"
+    fi
+
+    CONFIG+=(
+      -DCMAKE_CXX_FLAGS="-fsanitize=memory -stdlib=libc++ -isystem ${LLVM_BUILD_LIBCXX}/include -isystem ${LLVM_BUILD_LIBCXX}/include/c++/v1"
+    )
+
+    LLVM_LINKER_FLAGS="-lc++abi -Wl,--rpath=${LLVM_BUILD_LIBCXX}/lib -L${LLVM_BUILD_LIBCXX}/lib -fsanitize=memory -nostdinc++"
+  fi
+
+  cd "${LLVM_BUILD}"
+
   # Select components to build
-  CONFIG+=("-DLLVM_ENABLE_PROJECTS=clang;compiler-rt;libcxx;libcxxabi")
+  local ENABLED_LLVM_PROJECTS
+  ENABLED_LLVM_PROJECTS="clang"
 
   # cmake build
   if [[ "${enable_optimized}" == "1" && "${enable_debug}" != "1" ]]; then
@@ -214,17 +163,46 @@ configure_llvm() {
 
   # Remove unneeded targets
   CONFIG+=(
-    "-DLLVM_INCLUDE_EXAMPLES=OFF"
-    "-DCLANG_INCLUDE_TESTS=OFF"
-    "-DLLVM_INCLUDE_BENCHMARKS=OFF"
-    "-DBUILD_SHARED_LIBS=ON"
-    "-DLLVM_OPTIMIZED_TABLEGEN=ON"
-    "-DLLVM_TARGETS_TO_BUILD=X86"
+    "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF"
+    "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF"
+    "-DLLVM_INCLUDE_TESTS:BOOL=OFF"
+    "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=TRUE"
+    "-DLLVM_TARGETS_TO_BUILD:STRING=X86"
+    "-DLLVM_INSTALL_UTILS:BOOL=TRUE" # Install FileCheck and Not
+    "-DLLVM_PARALLEL_LINK_JOBS=1"
+    "-DCLANG_INCLUDE_TESTS:BOOL=OFF"
   )
 
+  if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then
+    # We have to build without libunwind if RTTI is disables
+    CONFIG+=("-DLLVM_ENABLE_PROJECTS=${ENABLED_LLVM_PROJECTS}")
+  else
+      CONFIG+=(
+          "-DLLVM_BUILD_LLVM_DYLIB:BOOL=ON"
+          "-DLLVM_LINK_LLVM_DYLIB:BOOL=ON"
+          "-DLLVM_BUILD_STATIC:BOOL=OFF"
+          "-DLIBCLANG_BUILD_STATIC:BOOL=OFF"
+      )
+      CONFIG+=("-DLLVM_ENABLE_PROJECTS=${ENABLED_LLVM_PROJECTS};libcxx;libcxxabi;libunwind")
+  fi
+
   if [[ -n ${SANITIZER_BUILD} ]]; then
-    CC="${SANITIZER_C_COMPILER}"
-    CXX="${SANITIZER_CXX_COMPILER}"
+    if [[ -n "${SANITIZER_C_COMPILER:-}" ]]; then
+      CC="${SANITIZER_C_COMPILER}"
+    fi
+    if [[ -n "${SANITIZER_CXX_COMPILER:-}" ]]; then
+      CXX="${SANITIZER_CXX_COMPILER}"
+    fi
+
+    if [[ "${SANITIZER_BUILD}" == "address" ]] ; then
+      CONFIG+=("-DLLVM_USE_SANITIZER=Address")
+    fi
+    if [[ "${SANITIZER_BUILD}" == "undefined" ]] ; then
+      CONFIG+=("-DLLVM_USE_SANITIZER=Undefined")
+    fi
+    if [[ "${SANITIZER_BUILD}" == "memory" ]] ; then
+      CONFIG+=("-DLLVM_USE_SANITIZER=MemoryWithOrigins")
+    fi
   fi
 
   (
@@ -236,21 +214,19 @@ configure_llvm() {
       export "CXX=${CXX}"
     fi
 
-    if [[ -n "${LDFLAGS:-}" ]]; then
-      export "LDFLAGS=${LLVM_LDFLAGS}"
+    if [[ -n "${LLVM_LINKER_FLAGS:-}" ]]; then
+      export "LDFLAGS=${LLVM_LINKER_FLAGS}"
     fi
 
-    cmake "${CONFIG[@]}" "${LLVM_SRC_BASE}/llvm"
+    cmake -G Ninja "${CONFIG[@]}" "${LLVM_SRC_BASE}/llvm"
   )
 }
 build_llvm() {
-  configure_llvm
-  (
-    if [[ "${SANITIZER_BUILD}" == "memory" ]]; then
-      ninja clang|| return 1
-      return 0
-    fi
+  # Skip building if already finished
+  [[ -f "${LLVM_BUILD}/.build_finished" ]] && return 0
 
+  configure_llvm || return 1
+  (
     if [[ -n "${CC:-}" ]]; then
       export "CC=${CC}"
     fi
@@ -259,45 +235,23 @@ build_llvm() {
       export "CXX=${CXX}"
     fi
 
-    if [[ -n "${LDFLAGS:-}" ]]; then
-      export "LDFLAGS=${LLVM_LDFLAGS}"
+    if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then
+      ninja clang || return 1
+    else
+      ninja || return 1
     fi
-
-    # Linking LLVM can require a lot of memory.
-    # First try multicore - if that doesn't work, try single core
-    (make "-j$(nproc)") || (make) || return 1
   ) || return 1
   touch "${LLVM_BUILD}/.build_finished"
+
 }
 
 install_llvm() {
-  if [[ "${SANITIZER_BUILD}" != "memory" ]]; then
-    cd "${LLVM_BUILD}"
-    make "-j$(nproc)" install
-    cp "${LLVM_BUILD_BIN}/FileCheck" "${LLVM_INSTALL}/bin/"
-    cp "${LLVM_BUILD_BIN}/not" "${LLVM_INSTALL}/bin/"
-
-    # Remove debug information from binaries
-    strip "${LLVM_INSTALL}/bin/"* || /bin/true
-    strip "${LLVM_INSTALL}/lib/libclang"* || /bin/true
-    strip "${LLVM_INSTALL}/lib/libLTO"* || /bin/true
-  else
-    # Handle memory sanitizer install
-    LLVM_PACKAGES=(
-      install-clang install-llvm-config install-llvm-objdump
-      install-llvm-link install-llvm-ar install-llvm-nm install-llvm-dis
-      install-clang-headers install-llvm-as
-      install-llvm-symbolizer install-LLVMSupport install-lli not FileCheck
-      install-llvm-headers
-      )
-
-    ninja "${LLVM_PACKAGES[@]}"
-
-    for i in $(ninja -t targets | grep install-LLVM | cut -d : -f 1); do ninja "$i"; done
-
-    cp "${LLVM_BUILD}/bin/FileCheck" "${LLVM_INSTALL}/bin/"
-    cp "${LLVM_BUILD}/bin/not" "${LLVM_INSTALL}/bin/"
-  fi
+  cd "${LLVM_BUILD}"
+  ninja install
+  # Remove debug information from binaries
+  strip "${LLVM_INSTALL}/bin/"* || /bin/true
+  strip "${LLVM_INSTALL}/lib/libclang"* || /bin/true
+  strip "${LLVM_INSTALL}/lib/libLTO"* || /bin/true
 
   touch "${LLVM_INSTALL}/.install_finished"
 }
@@ -307,7 +261,7 @@ is_installed_llvm() {
   (
     setup_build_variables_llvm
     # Check if the specific llvm-config exists
-    [[ -f "${LLVM_BUILD_BIN}/llvm-config" ]]
+    [[ -f "${LLVM_INSTALL}/bin/llvm-config" ]]
     [[ -f "${LLVM_INSTALL}/.install_finished" ]]
   ) || return 1
 }
@@ -327,9 +281,9 @@ setup_artifact_variables_llvm() {
 get_build_artifacts_llvm() {
   (
     setup_build_variables_llvm
+    if [[ "${SANITIZER_BUILD:-}" == "memory" ]]; then
+      echo "${LLVM_BUILD_LIBCXX}"
+    fi
     echo "${LLVM_INSTALL}"
-    echo "${LLVM_SRC_BASE}"
-    [[ "${sanitizer}" == "memory" ]] && echo "${SANITIZER_LLVM_UNINSTRUMENTED}"
-    [[ "${sanitizer}" == "memory" ]] && echo "${SANITIZER_LLVM_LIBCXX}"
   )
 }