Skip to content

Commit c581761

Browse files
alexreinkingalexreinking
authored
Add a tiny fuzz test framework (#9050)
* Add a Homebrew-compatible toolchain file * Manage libfuzzer flags from toolchain settings * Add stdlib-based fuzzer * Makefile support for fuzz tests * Make random numbers reproducible across platforms
1 parent 1d4002e commit c581761

12 files changed

Lines changed: 452 additions & 168 deletions

‎CMakePresets.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,20 @@
188188
"Halide_LLVM_ROOT": "/opt/homebrew/opt/llvm"
189189
}
190190
},
191+
{
192+
"name": "macOS-fuzz",
193+
"displayName": "macOS (Fuzz)",
194+
"description": "macOS fuzzing build",
195+
"inherits": "macOS",
196+
"toolchainFile": "${sourceDir}/cmake/toolchain.macos-homebrew.cmake",
197+
"cacheVariables": {
198+
"CMAKE_C_FLAGS": "-fsanitize=fuzzer-no-link -O1 -g3",
199+
"CMAKE_CXX_FLAGS": "-fsanitize=fuzzer-no-link -O1 -g3",
200+
"CMAKE_EXE_LINKER_FLAGS": "-fsanitize=fuzzer",
201+
"CMAKE_MODULE_LINKER_FLAGS": "-fsanitize=fuzzer",
202+
"CMAKE_SHARED_LINKER_FLAGS": "-fsanitize=fuzzer"
203+
}
204+
},
191205
{
192206
"name": "macOS-vcpkg",
193207
"inherits": [

‎Makefile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1284,6 +1284,7 @@ PERFORMANCE_TESTS = $(shell ls $(ROOT_DIR)/test/performance/*.cpp)
12841284
ERROR_TESTS = $(shell ls $(ROOT_DIR)/test/error/*.cpp)
12851285
WARNING_TESTS = $(shell ls $(ROOT_DIR)/test/warning/*.cpp)
12861286
RUNTIME_TESTS = $(shell ls $(ROOT_DIR)/test/runtime/*.cpp)
1287+
FUZZ_TESTS = $(filter-out %halide_fuzz_main.cpp, $(shell ls $(ROOT_DIR)/test/fuzz/*.cpp))
12871288
GENERATOR_EXTERNAL_TESTS := $(shell ls $(ROOT_DIR)/test/generator/*test.cpp)
12881289
GENERATOR_EXTERNAL_TEST_GENERATOR := $(shell ls $(ROOT_DIR)/test/generator/*_generator.cpp)
12891290
TUTORIALS = $(filter-out %_generate.cpp, $(shell ls $(ROOT_DIR)/tutorial/*.cpp))
@@ -1297,6 +1298,7 @@ test_performance: $(PERFORMANCE_TESTS:$(ROOT_DIR)/test/performance/%.cpp=perform
12971298
test_error: $(ERROR_TESTS:$(ROOT_DIR)/test/error/%.cpp=error_%)
12981299
test_warning: $(WARNING_TESTS:$(ROOT_DIR)/test/warning/%.cpp=warning_%)
12991300
test_runtime: $(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=runtime_%)
1301+
test_fuzz: $(FUZZ_TESTS:$(ROOT_DIR)/test/fuzz/%.cpp=fuzz_%)
13001302
test_tutorial: $(TUTORIALS:$(ROOT_DIR)/tutorial/%.cpp=tutorial_%)
13011303
test_valgrind: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=valgrind_%)
13021304
test_avx512: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=avx512_%)
@@ -1392,6 +1394,7 @@ build_tests: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=$(BIN_DIR)/c
13921394
$(ERROR_TESTS:$(ROOT_DIR)/test/error/%.cpp=$(BIN_DIR)/error_%) \
13931395
$(WARNING_TESTS:$(ROOT_DIR)/test/warning/%.cpp=$(BIN_DIR)/warning_%) \
13941396
$(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=$(BIN_DIR)/runtime_%) \
1397+
$(FUZZ_TESTS:$(ROOT_DIR)/test/fuzz/%.cpp=$(BIN_DIR)/fuzz_%) \
13951398
$(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_aottest.cpp=$(BIN_DIR)/$(TARGET)/generator_aot_%) \
13961399
$(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_jittest.cpp=$(BIN_DIR)/generator_jit_%) \
13971400
$(MULLAPUDI2016_TESTS:$(ROOT_DIR)/test/autoschedulers/mullapudi2016/%.cpp=$(BIN_DIR)/mullapudi2016_%) \
@@ -1470,6 +1473,9 @@ $(BIN_DIR)/$(TARGET)/correctness_opencl_runtime: $(ROOT_DIR)/test/correctness/op
14701473
$(BIN_DIR)/performance_%: $(ROOT_DIR)/test/performance/%.cpp $(TEST_DEPS)
14711474
$(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE) $< -I$(INCLUDE_DIR) -I$(ROOT_DIR)/src/runtime -I$(ROOT_DIR)/test/common $(TEST_LD_FLAGS) -o $@
14721475

1476+
$(BIN_DIR)/fuzz_%: $(ROOT_DIR)/test/fuzz/%.cpp $(ROOT_DIR)/test/fuzz/halide_fuzz_main.cpp $(ROOT_DIR)/test/fuzz/fuzz_helpers.h $(ROOT_DIR)/test/fuzz/halide_fuzz_main.h $(TEST_DEPS)
1477+
$(CXX) $(TEST_CXX_FLAGS) -I$(ROOT_DIR)/src/runtime -I$(ROOT_DIR)/test/common $(OPTIMIZE_FOR_BUILD_TIME) $(filter %.cpp,$^) -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@ -DHALIDE_FUZZER_BACKEND=0
1478+
14731479
# Error tests that link against libHalide
14741480
$(BIN_DIR)/error_%: $(ROOT_DIR)/test/error/%.cpp $(TEST_DEPS)
14751481
$(CXX) $(TEST_CXX_FLAGS) -I$(ROOT_DIR)/src/runtime -I$(ROOT_DIR)/test/common $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@
@@ -2063,6 +2069,11 @@ quiet_correctness_%: $(BIN_DIR)/correctness_%
20632069
@-mkdir -p $(TMP_DIR)
20642070
@cd $(TMP_DIR) ; ( $(CURDIR)/$< 2>stderr_$*.txt > stdout_$*.txt && echo -n . ) || ( echo ; echo FAILED TEST: $* ; cat stdout_$*.txt stderr_$*.txt ; false )
20652071

2072+
fuzz_%: $(BIN_DIR)/fuzz_%
2073+
@-mkdir -p $(TMP_DIR)
2074+
cd $(TMP_DIR) ; $(CURDIR)/$<
2075+
@-echo
2076+
20662077
valgrind_%: $(BIN_DIR)/correctness_%
20672078
@-mkdir -p $(TMP_DIR)
20682079
cd $(TMP_DIR) ; valgrind --error-exitcode=-1 $(CURDIR)/$<

‎cmake/toolchain.macos-homebrew.cmake‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
cmake_minimum_required(VERSION 3.28)
2+
3+
set(CMAKE_C_COMPILER /opt/homebrew/opt/llvm@21/bin/clang)
4+
set(CMAKE_CXX_COMPILER /opt/homebrew/opt/llvm@21/bin/clang++)
5+
6+
if (NOT DEFINED CMAKE_SYSROOT)
7+
execute_process(
8+
COMMAND xcrun --show-sdk-path
9+
OUTPUT_VARIABLE CMAKE_SYSROOT
10+
OUTPUT_STRIP_TRAILING_WHITESPACE
11+
COMMAND_ERROR_IS_FATAL ANY)
12+
endif ()
13+
14+
set(CMAKE_SYSROOT "${CMAKE_SYSROOT}" CACHE PATH "")
15+
16+
if (NOT DEFINED XC_TOOLCHAIN_PATH)
17+
execute_process(
18+
COMMAND xcrun --find clang
19+
OUTPUT_VARIABLE XC_TOOLCHAIN_PATH
20+
OUTPUT_STRIP_TRAILING_WHITESPACE
21+
COMMAND_ERROR_IS_FATAL ANY)
22+
cmake_path(SET XC_TOOLCHAIN_PATH NORMALIZE "${XC_TOOLCHAIN_PATH}/../../..")
23+
endif ()
24+
25+
set(XC_TOOLCHAIN_PATH "${XC_TOOLCHAIN_PATH}" CACHE PATH "")
26+
27+
set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES
28+
"${CMAKE_SYSROOT}/usr/include"
29+
"${XC_TOOLCHAIN_PATH}/usr/include"
30+
"${CMAKE_SYSROOT}/System/Library/Frameworks"
31+
"${CMAKE_SYSROOT}/System/Library/SubFrameworks"
32+
)
33+
34+
set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES
35+
"${CMAKE_SYSROOT}/usr/include/c++/v1"
36+
${CMAKE_C_STANDARD_INCLUDE_DIRECTORIES}
37+
)

‎run-clang-tidy.sh‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -111,27 +111,7 @@ export CMAKE_EXPORT_COMPILE_COMMANDS=ON
111111
export Halide_LLVM_ROOT="${CLANG_TIDY_LLVM_INSTALL_DIR}"
112112

113113
if [[ $(${CC} --version) =~ .*Homebrew.* ]]; then
114-
# Homebrew clang 21 is badly misconfigured and needs help finding the
115-
# system headers, even though it uses system libc++ by default.
116-
SDKROOT="$(xcrun --show-sdk-path)"
117-
# TOOLCHAINROOT="$(xcrun --show-toolchain-path)"
118-
TOOLCHAINROOT="$(cd "$(dirname "$(xcrun --find clang)")"/../.. && pwd)"
119-
RCDIR="$(xcrun clang -print-resource-dir)"
120-
cat >"${CLANG_TIDY_BUILD_DIR}/toolchain.cmake" <<EOF
121-
set(CMAKE_SYSROOT "${SDKROOT}")
122-
set(CMAKE_C_STANDARD_INCLUDE_DIRECTORIES
123-
"${RCDIR}/include"
124-
"${SDKROOT}/usr/include"
125-
"${TOOLCHAINROOT}/usr/include"
126-
"${SDKROOT}/System/Library/Frameworks"
127-
"${SDKROOT}/System/Library/SubFrameworks"
128-
)
129-
set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES
130-
"${SDKROOT}/usr/include/c++/v1"
131-
\${CMAKE_C_STANDARD_INCLUDE_DIRECTORIES}
132-
)
133-
EOF
134-
export CMAKE_TOOLCHAIN_FILE="${CLANG_TIDY_BUILD_DIR}/toolchain.cmake"
114+
export CMAKE_TOOLCHAIN_FILE="${ROOT_DIR}/cmake/toolchain.macos-homebrew.cmake"
135115
fi
136116

137117
echo Configuring Halide...

‎src/Bounds.cpp‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,11 @@ class Bounds : public IRVisitor {
198198
void log_line(Args &&...args) {
199199
debug(0) << self->log_spaces();
200200
// C++17 right fold
201-
(debug(0) << ... << args) << "\n";
201+
auto dump = [](auto &&arg) {
202+
debug(0) << arg;
203+
};
204+
(dump(args), ...);
205+
debug(0) << "\n";
202206
}
203207

204208
~BoundsLogger() {
@@ -2144,7 +2148,11 @@ class BoxesTouched : public IRGraphVisitor {
21442148
void log_line(Args &&...args) {
21452149
debug(0) << self->log_spaces();
21462150
// C++17 right fold
2147-
(debug(0) << ... << args) << "\n";
2151+
auto dump = [](auto &&arg) {
2152+
debug(0) << arg;
2153+
};
2154+
(dump(args), ...);
2155+
debug(0) << "\n";
21482156
}
21492157

21502158
BoxesTouchedLogger(BoxesTouched *self, const char *pretty_function)

‎test/CMakeLists.txt‎

Lines changed: 1 addition & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -67,36 +67,7 @@ endif ()
6767

6868
# FIXME: failing_with_issue is dead code :)
6969

70-
# Ensure that basic sanitizer flags are supported;
71-
# - Address sanitizer is often used in conjunction with fuzzing as it will detect
72-
# common high severity bugs. This sanitizer is used as a "default" for fuzzing
73-
# when the sanitizer isn't otherwise specified.
74-
# - Fuzzer sanitizer will link against libfuzzer and is currently only supported
75-
# on clang/msvc and isn't supported with GCC. If you need to use these fuzzers
76-
# with a GCC based project you should consider looking into the LIB_FUZZING_ENGINE
77-
# env variable defined in `test/fuzz/CMakeLists.txt`.
78-
set(CMAKE_REQUIRED_LINK_OPTIONS "-fsanitize=fuzzer,address")
79-
set(CMAKE_REQUIRED_FLAGS "-fsanitize=fuzzer-no-link,address")
80-
check_cxx_source_compiles([[
81-
#include <cstdint>
82-
#include <cstddef>
83-
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, std::size_t Size) {
84-
return 0;
85-
}
86-
]] HAS_FUZZ_FLAGS)
87-
88-
if (NOT HAS_FUZZ_FLAGS)
89-
message(VERBOSE "Compiler does not support libfuzzer sanitizer.")
90-
else ()
91-
message(VERBOSE "Compiler supports libfuzzer sanitizer.")
92-
endif ()
93-
94-
# Note that we want to default WITH_TEST_FUZZ to OFF, even if HAS_FUZZ_FLAGS
95-
# is true: just because our compiler supports fuzzing doesn't mean we want to
96-
# build the fuzz tests, because they won't really build properly without the
97-
# right preset specified.
98-
Halide_feature(WITH_TEST_FUZZ "Build fuzz tests" AUTO
99-
DEPENDS HAS_FUZZ_FLAGS)
70+
Halide_feature(WITH_TEST_FUZZ "Build fuzz tests" ON)
10071
if (WITH_TEST_FUZZ)
10172
add_subdirectory(fuzz)
10273
endif ()

‎test/fuzz/CMakeLists.txt‎

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
if (NOT LIB_FUZZING_ENGINE)
2+
set(LIB_FUZZING_ENGINE "$ENV{LIB_FUZZING_ENGINE}")
3+
endif ()
4+
5+
set(LIB_FUZZING_ENGINE "${LIB_FUZZING_ENGINE}"
6+
CACHE STRING "Extra link libraries arguments for fuzz tests.")
7+
18
tests(GROUPS fuzz
29
SOURCES
310
bounds.cpp
@@ -15,25 +22,31 @@ tests(GROUPS fuzz
1522
USE_EXIT_CODE_ONLY
1623
)
1724

25+
# Check whether the configured toolchain can compile a libfuzzer test harness.
26+
# If this fails, the user likely needs to set up their environment to use a
27+
# compatible fuzzing engine (e.g. by setting up a toolchain file).
28+
check_cxx_source_compiles([[
29+
#include <cstdint>
30+
#include <cstddef>
31+
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, std::size_t Size) {
32+
return 0;
33+
}
34+
]] HAVE_LIBFUZZER_FLAGS)
1835

19-
# By default we are going to use the libfuzzer engine. However if
20-
# LIB_FUZZING_ENGINE is declared you can override the fuzzing engine to one of;
21-
# - Centipede
22-
# - Hongfuzz
23-
# - AFL++
24-
# - etc.
25-
set(LIB_FUZZING_ENGINE "$ENV{LIB_FUZZING_ENGINE}"
26-
CACHE STRING "Compiler flags necessary to link the fuzzing engine of choice e.g. libfuzzer, afl etc.")
36+
add_library(Halide_fuzz INTERFACE)
37+
add_library(Halide::fuzz ALIAS Halide_fuzz)
2738

28-
foreach(fuzzer "fuzz_bounds" "fuzz_cse")
29-
target_link_libraries(${fuzzer} PRIVATE Halide::Halide)
39+
if (NOT HAVE_LIBFUZZER_FLAGS)
40+
if (LIB_FUZZING_ENGINE)
41+
message(FATAL_ERROR "Cannot set LIB_FUZZING_ENGINE when not building with -fsanitize=fuzzer or a compatible fuzzing engine.")
42+
endif ()
43+
target_sources(Halide_fuzz INTERFACE halide_fuzz_main.cpp halide_fuzz_main.h)
44+
target_compile_definitions(Halide_fuzz INTERFACE HALIDE_FUZZER_BACKEND=HALIDE_FUZZER_BACKEND_STDLIB)
45+
else ()
46+
target_link_libraries(Halide_fuzz INTERFACE ${LIB_FUZZING_ENGINE})
47+
target_compile_definitions(Halide_fuzz INTERFACE HALIDE_FUZZER_BACKEND=HALIDE_FUZZER_BACKEND_LIBFUZZER)
48+
endif ()
3049

31-
# Allow OSS-fuzz to manage flags directly
32-
if (LIB_FUZZING_ENGINE)
33-
target_link_libraries(${fuzzer} PRIVATE "${LIB_FUZZING_ENGINE}")
34-
else ()
35-
# By default just build with address-sanitizers/libfuzzer for local testing
36-
target_compile_options(${fuzzer} PRIVATE -fsanitize=fuzzer-no-link)
37-
target_link_options(${fuzzer} PRIVATE -fsanitize=fuzzer)
38-
endif ()
39-
endforeach()
50+
foreach (fuzzer IN LISTS TEST_NAMES)
51+
target_link_libraries("${fuzzer}" PRIVATE Halide::fuzz)
52+
endforeach ()

0 commit comments

Comments
 (0)