blob: 7eb8b4fab4d2efc15c22a91e449051f9912764e9 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
#!/bin/bash
set -e
set -u
DIR="$(cd "$(dirname "$0")" && pwd)"
coverage_setup() {
local build_dir="$1"
# Zero coverage for any file, e.g. previous tests
lcov -q --directory "${build_dir}" --no-external --zerocounters
# Create a baseline by capturing any file used for compilation, no execution yet
lcov -q --rc lcov_branch_coverage=1 --directory "${build_dir}" --base-directory="${KLEE_SRC}" --no-external --capture --initial --output-file coverage_base.info
lcov -q --rc lcov_branch_coverage=1 --remove coverage_base.info 'test/*' --output-file coverage_base.info
lcov -q --rc lcov_branch_coverage=1 --remove coverage_base.info 'unittests/*' --output-file coverage_base.info
}
coverage_update() {
tags="$2"
codecov_suffix=(${tags// /})
build_dir="$1"
# Create report
# (NOTE: "--rc lcov_branch_coverage=1" needs to be added in all calls, otherwise branch coverage gets dropped)
lcov -q --rc lcov_branch_coverage=1 --directory "${build_dir}" --base-directory="${KLEE_SRC}" --no-external --capture --output-file coverage.info
# Exclude uninteresting coverage goals (LLVM, googletest, and KLEE system and unit tests)
lcov -q --rc lcov_branch_coverage=1 --remove coverage.info 'test/*' --output-file coverage.info
lcov -q --rc lcov_branch_coverage=1 --remove coverage.info 'unittests/*' --output-file coverage.info
# Combine baseline and measured coverage
lcov -q --rc lcov_branch_coverage=1 -a coverage_base.info -a coverage.info -o coverage_all.info."${codecov_suffix}"
# Debug info
lcov -q --rc lcov_branch_coverage=1 --list coverage_all.info."${codecov_suffix}"
}
run_tests() {
build_dir="$1"
KLEE_SRC="$(cd "${DIR}"/../../ && pwd)"
# TODO change to pinpoint specific directory
cd "${build_dir}"
# Remove klee from PATH
export PATH=${PATH%":/home/klee/klee_build/bin"}
if which klee; then
return 1 # should not happen
fi
###############################################################################
# Unit tests
###############################################################################
# Prepare coverage information if COVERAGE is set
if [ "${COVERAGE}" -eq 1 ]; then
coverage_setup "${build_dir}"
fi
make unittests
# Generate and upload coverage if COVERAGE is set
if [ "${COVERAGE}" -eq 1 ]; then
coverage_update "${build_dir}" "unittests"
fi
###############################################################################
# lit tests
###############################################################################
if [ "${COVERAGE}" -eq 1 ]; then
coverage_setup "${build_dir}"
fi
make systemtests || return 1
# If metaSMT is the only solver, then rerun lit tests with non-default metaSMT backends
if [ "X${SOLVERS}" == "XmetaSMT" ]; then
base_path="$(python3 -m site --user-base)"
export PATH="$PATH:${base_path}/bin"
available_metasmt_backends="btor stp z3 yices2 cvc4"
for backend in $available_metasmt_backends; do
if [ "X${METASMT_DEFAULT}" != "X$backend" ]; then
lit -v --param klee_opts=-metasmt-backend="$backend" --param kleaver_opts=-metasmt-backend="$backend" test/
fi
done
fi
# Generate and upload coverage if COVERAGE is set
if [ "${COVERAGE}" -eq 1 ]; then
coverage_update "${build_dir}" "systemtests"
fi
}
function upload_coverage() {
file="$1"
tags="$2"
cd /home/klee/klee_src
bash <(curl -s https://codecov.io/bash) -X gcov -R /tmp/klee_src/ -f /home/klee/klee_build/coverage_all.info."${file}" -F "$tags"
}
function run_docker() {
docker_arguments=(docker run -u root --cap-add SYS_PTRACE -t -e SOLVERS -e METASMT_DEFAULT)
script_arguments=("--debug" '"/tmp/klee_build"*')
if [[ "${COVERAGE}" -eq 1 ]]; then
script_arguments+=("--coverage")
fi
if [[ "${UPLOAD_COVERAGE}" -eq 1 ]]; then
docker_arguments+=($(bash <(curl -s https://codecov.io/env)))
script_arguments+=("--upload-coverage")
fi
# Run the image that was build last with extended capabilities to allow tracing tests
"${docker_arguments[@]}" "$(docker images -q | head -n 1)" /bin/bash -i -c "ulimit -s 16384; source /home/klee/.bashrc; export; /tmp/klee_src/scripts/build/run-tests.sh ${script_arguments[*]}"
}
main() {
local NAME
NAME=$(basename "${0}")
local RUN_DOCKER=0
local COVERAGE=0
local UPLOAD_COVERAGE=0
local directory=""
for i in "$@" ;do
case $i in
--debug)
set -x
;;
--run-docker)
RUN_DOCKER=1
;;
--coverage)
COVERAGE=1
;;
--upload-coverage)
UPLOAD_COVERAGE=1
;;
-*)
echo "${NAME}: unknown argument: $i"
exit 1
;;
*)
directory="${i}"
;;
esac
done
if [[ "${RUN_DOCKER}" -eq 1 ]]; then
run_docker "${COVERAGE}"
return 0
fi
run_tests "${directory}"
# FIXME Enable separated coverage tags again
if [[ "${UPLOAD_COVERAGE}" -eq 1 ]]; then
upload_coverage systemtests systemtests_unittests
upload_coverage unittests systemtests_unittests
fi
}
main "$@"
|
