about summary refs log tree commit diff homepage
path: root/scripts/build/run-tests.sh
blob: 4b9b972e92b5a3827158d36ff965fbf5c98f9297 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
#!/bin/bash
set -e
set -u
DIR="$(cd "$(dirname "$0")" && pwd)"


coverage_setup() {
  local build_dir="$1"
  # Zero coverage for any file, e.g. previous tests
  lcov -q --directory "${build_dir}" --no-external --zerocounters
  # Create a baseline by capturing any file used for compilation, no execution yet
  lcov -q --rc lcov_branch_coverage=1 --directory "${build_dir}" --base-directory="${KLEE_SRC}" --no-external --capture --initial --output-file coverage_base.info
  lcov -q --rc lcov_branch_coverage=1 --remove coverage_base.info 'test/*' --output-file coverage_base.info
  lcov -q --rc lcov_branch_coverage=1 --remove coverage_base.info 'unittests/*' --output-file coverage_base.info
}

coverage_update() {
  tags="$2"
  codecov_suffix=(${tags// /})
  build_dir="$1"
  # Create report
  # (NOTE: "--rc lcov_branch_coverage=1" needs to be added in all calls, otherwise branch coverage gets dropped)
  lcov -q --rc lcov_branch_coverage=1 --directory "${build_dir}" --base-directory="${KLEE_SRC}" --no-external --capture --output-file coverage.info
  # Exclude uninteresting coverage goals (LLVM, googletest, and KLEE system and unit tests)
  lcov -q --rc lcov_branch_coverage=1 --remove coverage.info 'test/*' --output-file coverage.info
  lcov -q --rc lcov_branch_coverage=1 --remove coverage.info 'unittests/*' --output-file coverage.info
  # Combine baseline and measured coverage
  lcov -q --rc lcov_branch_coverage=1 -a coverage_base.info -a coverage.info -o coverage_all.info."${codecov_suffix}"
  # Debug info
  lcov -q --rc lcov_branch_coverage=1 --list coverage_all.info."${codecov_suffix}"
}

run_tests() {
  build_dir="$1"
  KLEE_SRC="$(cd "${DIR}"/../../ && pwd)"

  # TODO change to pinpoint specific directory
  cd "${build_dir}"

  # Remove klee from PATH
  export PATH=${PATH/":/home/klee/klee_build/bin"/}
  if which klee; then
    return 1 # should not happen
  fi

  ###############################################################################
  # Unit tests
  ###############################################################################
  # Prepare coverage information if COVERAGE is set
  if [ "${COVERAGE}" -eq 1 ]; then
    coverage_setup "${build_dir}"
  fi
  make unittests
  
  # Generate and upload coverage if COVERAGE is set
  if [ "${COVERAGE}" -eq 1 ]; then
    coverage_update "${build_dir}" "unittests"
  fi

  ###############################################################################
  # lit tests
  ###############################################################################
  if [ "${COVERAGE}" -eq 1 ]; then
    coverage_setup "${build_dir}"
  fi

  make systemtests || return 1
  
  # If metaSMT is the only solver, then rerun lit tests with non-default metaSMT backends
  if [ "X${SOLVERS}" == "XmetaSMT" ]; then
    base_path="$(python3 -m site --user-base)"
    export PATH="$PATH:${base_path}/bin"
    available_metasmt_backends="btor stp z3 yices2 cvc4"
    for backend in $available_metasmt_backends; do
      if [ "X${METASMT_DEFAULT}" != "X$backend" ]; then
        lit -v --param klee_opts=-metasmt-backend="$backend" --param kleaver_opts=-metasmt-backend="$backend" test/
      fi
    done
  fi
  
  # Generate and upload coverage if COVERAGE is set
  if [ "${COVERAGE}" -eq 1 ]; then
    coverage_update "${build_dir}" "systemtests"
  fi
}



function upload_coverage() {
  file="$1"
  tags="$2"
  cd /home/klee/klee_src
  bash <(curl -s https://codecov.io/bash) -X gcov -R /tmp/klee_src/ -f /home/klee/klee_build/coverage_all.info."${file}" -F "$tags"
}

function run_docker() {
 docker_arguments=(docker run -u root --cap-add SYS_PTRACE -t -e SOLVERS -e METASMT_DEFAULT)
 script_arguments=("--debug" '"/tmp/klee_build"*')
 if [[ "${COVERAGE}" -eq 1 ]]; then
   script_arguments+=("--coverage")
 fi
 
 if [[ "${UPLOAD_COVERAGE}" -eq 1 ]]; then
   docker_arguments+=($(bash <(curl -s https://codecov.io/env)))
   script_arguments+=("--upload-coverage")
 fi

 # Run the image that was build last with extended capabilities to allow tracing tests
 "${docker_arguments[@]}" "$(docker images -q | head -n 1)" /bin/bash -i -c "ulimit -s 16384; source /home/klee/.bashrc; export; /tmp/klee_src/scripts/build/run-tests.sh ${script_arguments[*]}"
}

main() {
  local NAME
  NAME=$(basename "${0}")
  local RUN_DOCKER=0
  local COVERAGE=0
  local UPLOAD_COVERAGE=0
  local directory=""

  for i in "$@" ;do
  case $i in
    --debug)
      set -x
    ;;
    --run-docker)
    RUN_DOCKER=1
    ;;
    --coverage)
    COVERAGE=1
    ;;
    --upload-coverage)
    UPLOAD_COVERAGE=1
    ;;
    -*)
      echo "${NAME}: unknown argument: $i"
      exit 1
    ;;
    *)
    directory="${i}"
    ;;
  esac
  done


  if [[ "${RUN_DOCKER}" -eq 1 ]]; then
    run_docker "${COVERAGE}"
    return 0
  fi

  run_tests "${directory}"
  
  # FIXME Enable separated coverage tags again
  if [[ "${UPLOAD_COVERAGE}" -eq 1 ]]; then
    upload_coverage systemtests systemtests_unittests
    upload_coverage unittests systemtests_unittests
  fi
 }

main "$@"