Refactor CI workflows and improve test configuration (#9010)

* Add Halide_BUILDING_IN_CI feature
* On CI, only build simd_op_check tests for host target
* Move "Check CMake file lists" to ubuntu-slim runner
* Consolidate test sources; use multithreaded label for RUN_SERIAL
* Correct multithreaded test groupings
* Skip gpu_metal_completion_handler_error_check on Apple Silicon
* Move clang-tidy to separate workflow and filter paths
* Appease clang-format

Author: alexreinking

.github/workflows/presubmit-clang-tidy.yml

@@ -0,0 +1,35 @@
+name: Clang-Tidy
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened ]
+    paths:
+      - '**.h'
+      - '**.c'
+      - '**.cpp'
+      - 'run-clang-tidy.sh'
+      - '**.clang-tidy'
+      - '.github/workflows/presubmit-clang-tidy.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check_clang_tidy:
+    name: Check clang-tidy
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # We intentionally don't use VCPKG here so that we get some
+      # notification that the ecosystem is moving on without us.
+      - name: Install clang-tidy & dependencies
+        run: brew install flatbuffers llvm@21 lld@21 pybind11 wabt
+
+      - name: Run clang-tidy
+        run: ./run-clang-tidy.sh
+        env:
+          CLANG_TIDY_LLVM_INSTALL_DIR: /opt/homebrew/opt/llvm@21

.github/workflows/presubmit.yml

@@ -4,67 +4,55 @@ on:
   # We don't want 'review_requested' (that's redundant to the ones below for our purposes)
   pull_request:
     types: [ opened, synchronize, reopened ]
-    paths:
-      - '**.h'
-      - '**.c'
-      - '**.cpp'
-      - '**CMakeLists.txt'
-      - '**.cmake'
-      - 'run-clang-tidy.sh'
-      - 'run-clang-format.sh'
-      - '.github/workflows/**'
-      - '**.clang-tidy'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 permissions:
   contents: read
 
 jobs:
   check_clang_format:
-    name: Check clang-format and ruff
-    runs-on: macos-14
+    name: Check clang-format
+    runs-on: ubuntu-slim
     steps:
       - uses: actions/checkout@v4
 
       - name: Install clang-format
-        run: brew install llvm@21
+        run: |
+          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+          echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+          sudo apt-get update
+          sudo apt-get install -y clang-format-21
 
       - name: Check clang-format
         run: ./run-clang-format.sh -c
         env:
-          CLANG_FORMAT_LLVM_INSTALL_DIR: /opt/homebrew/opt/llvm@21
+          CLANG_FORMAT_LLVM_INSTALL_DIR: /usr/lib/llvm-21
 
-      - uses: astral-sh/ruff-action@v3
-
-  check_clang_tidy:
-    name: Check clang-tidy
-    runs-on: macos-14
+  check_ruff:
+    name: Check ruff
+    runs-on: ubuntu-slim
     steps:
       - uses: actions/checkout@v4
-
-      # We intentionally don't use VCPKG here so that we get some
-      # notification that the ecosystem is moving on without us.
-      - name: Install clang-tidy & dependencies
-        run: brew install flatbuffers llvm@21 lld@21 pybind11 wabt
-
-      - name: Run clang-tidy
-        run: ./run-clang-tidy.sh
-        env:
-          CLANG_TIDY_LLVM_INSTALL_DIR: /opt/homebrew/opt/llvm@21
+      - uses: astral-sh/ruff-action@v3
 
   check_cmake_file_lists:
     name: Check CMake file lists
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-slim
     steps:
       - uses: actions/checkout@v4
       - name: Run test sources check
         run: |
           check_dir() {
             local dir="$1"
+            echo "Checking $dir"
             (
               cd "$dir" &&
               comm -23 \
                 <(find . -maxdepth 1 -type f \( -name '*.c' -o -name '*.cpp' \) -printf '%f\n' | sort) \
-                <(grep -P '^\s*#?\s*[A-Za-z0-9_.]+$' CMakeLists.txt | tr -d '# ' | sort) \
+                <(grep -oP '\b[A-Za-z0-9_]+\.(?:cpp|c)\b' CMakeLists.txt | sort -u) \
                 | tee missing_files &&
               [ ! -s missing_files ]
             )

.github/workflows/testing-arm-linux.yml

@@ -3,6 +3,20 @@ name: ARM Linux
 on:
   pull_request:
     types: [ opened, synchronize, reopened ]
+    paths-ignore:
+      - "doc/**"
+      - "README.md"
+      - "CODE_OF_CONDUCT.md"
+      - "LICENSE.txt"
+      - ".gitignore"
+      - ".gitattributes"
+      - ".gitmodules"
+      - ".lldbinit"
+      - ".github/**"
+      - "!.github/workflows/testing-arm-linux.yml"
+      - "packaging/**"
+      - "Makefile"
+      - "Makefile.inc"
   workflow_dispatch:
 
 concurrency:

.github/workflows/testing-linux.yml

@@ -3,6 +3,20 @@ name: Linux
 on:
   pull_request:
     types: [ opened, synchronize, reopened ]
+    paths-ignore:
+      - "doc/**"
+      - "README.md"
+      - "CODE_OF_CONDUCT.md"
+      - "LICENSE.txt"
+      - ".gitignore"
+      - ".gitattributes"
+      - ".gitmodules"
+      - ".lldbinit"
+      - ".github/**"
+      - "!.github/workflows/testing-linux.yml"
+      - "packaging/**"
+      - "Makefile"
+      - "Makefile.inc"
   workflow_dispatch:
 
 concurrency:

.github/workflows/testing-make.yml

@@ -21,7 +21,7 @@ on:
       - "python_bindings/**"
 
       - "pyproject.toml"
-      - "uv.lock"
+      # Don't ignore uv.lock as it influences dependency resolution
 
       - "README.md"
       - "CODE_OF_CONDUCT.md"
@@ -30,11 +30,18 @@ on:
       - ".gitattributes"
       - ".gitmodules"
       - ".lldbinit"
+      - "run-clang-format.sh"
+      - "run-clang-tidy.sh"
+      - "**.clang-tidy"
+      - "**.py"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build_make:
     name: build on ${{ matrix.os }}

.github/workflows/testing-windows.yml

@@ -3,6 +3,20 @@ name: Windows
 on:
   pull_request:
     types: [ opened, synchronize, reopened ]
+    paths-ignore:
+      - "doc/**"
+      - "README.md"
+      - "CODE_OF_CONDUCT.md"
+      - "LICENSE.txt"
+      - ".gitignore"
+      - ".gitattributes"
+      - ".gitmodules"
+      - ".lldbinit"
+      - ".github/**"
+      - "!.github/workflows/testing-windows.yml"
+      - "packaging/**"
+      - "Makefile"
+      - "Makefile.inc"
   workflow_dispatch:
 
 concurrency:

CMakeLists.txt

@@ -206,6 +206,8 @@ find_package(PNG)
 ##
 # Optional features. These settings are defined early so that subdirectories see a consistent view
 
+Halide_feature(Halide_BUILDING_IN_CI "Enable if building in CI" OFF ADVANCED)
+
 Halide_feature(Halide_ENABLE_EXCEPTIONS "Enable exceptions in Halide" ON)
 Halide_feature(Halide_ENABLE_RTTI "Enable RTTI in Halide" ON
                DEPENDS LLVM_ENABLE_RTTI)

CMakePresets.json

@@ -23,7 +23,8 @@
         "CMAKE_BUILD_TYPE": "RelWithDebInfo",
         "FETCHCONTENT_FULLY_DISCONNECTED": "ON",
         "VCPKG_CHAINLOAD_TOOLCHAIN_FILE": "${sourceDir}/cmake/toolchain.${presetName}.cmake",
-        "VCPKG_MANIFEST_FEATURES": "developer"
+        "VCPKG_MANIFEST_FEATURES": "developer",
+        "Halide_BUILDING_IN_CI": "ON"
       }
     },
     {

Makefile

@@ -281,7 +281,7 @@ TEST_CXX_FLAGS += -DLLVM_VERSION=$(LLVM_VERSION_TIMES_10)
 TEST_CXX_FLAGS += -fvisibility=hidden -fvisibility-inlines-hidden
 
 # In the tests, enable the debug() and internal_assert() macros
-TEST_CXX_FLAGS += -DHALIDE_KEEP_MACROS
+TEST_CXX_FLAGS += -DHALIDE_KEEP_MACROS=
 
 # gcc 4.8 fires a bogus warning on old versions of png.h
 ifneq (,$(findstring g++,$(CXX_VERSION)))
@@ -1282,8 +1282,9 @@ GENERATOR_EXTERNAL_TESTS := $(shell ls $(ROOT_DIR)/test/generator/*test.cpp)
 GENERATOR_EXTERNAL_TEST_GENERATOR := $(shell ls $(ROOT_DIR)/test/generator/*_generator.cpp)
 TUTORIALS = $(filter-out %_generate.cpp, $(shell ls $(ROOT_DIR)/tutorial/*.cpp))
 MULLAPUDI2016_TESTS = $(shell ls $(ROOT_DIR)/test/autoschedulers/mullapudi2016/*.cpp)
-LI2018_TESTS = $(shell ls $(ROOT_DIR)/test/autoschedulers/li2018/test.cpp)
-ADAMS2019_TESTS = $(shell ls $(ROOT_DIR)/test/autoschedulers/adams2019/test.cpp)
+LI2018_TESTS = $(filter-out %_generator.cpp, $(shell ls $(ROOT_DIR)/test/autoschedulers/li2018/*.cpp))
+ADAMS2019_TESTS = $(filter-out %_generator.cpp, $(shell ls $(ROOT_DIR)/test/autoschedulers/adams2019/*.cpp))
+COMMON_AUTOSCHEDULER_TESTS = $(shell ls $(ROOT_DIR)/test/autoschedulers/common/*.cpp)
 
 test_correctness: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=quiet_correctness_%) $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.c=quiet_correctness_%)
 test_performance: $(PERFORMANCE_TESTS:$(ROOT_DIR)/test/performance/%.cpp=performance_%)
@@ -1293,7 +1294,7 @@ test_runtime: $(RUNTIME_TESTS:$(ROOT_DIR)/test/runtime/%.cpp=runtime_%)
 test_tutorial: $(TUTORIALS:$(ROOT_DIR)/tutorial/%.cpp=tutorial_%)
 test_valgrind: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=valgrind_%)
 test_avx512: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=avx512_%)
-test_autoschedulers: test_mullapudi2016 test_li2018 test_adams2019
+test_autoschedulers: test_mullapudi2016 test_li2018 test_adams2019 test_common_autoscheduler
 test_auto_schedule: test_autoschedulers
 
 .PHONY: test_correctness_multi_gpu
@@ -1389,7 +1390,8 @@ build_tests: $(CORRECTNESS_TESTS:$(ROOT_DIR)/test/correctness/%.cpp=$(BIN_DIR)/c
 	$(GENERATOR_EXTERNAL_TESTS:$(ROOT_DIR)/test/generator/%_jittest.cpp=$(BIN_DIR)/generator_jit_%) \
 	$(MULLAPUDI2016_TESTS:$(ROOT_DIR)/test/autoschedulers/mullapudi2016/%.cpp=$(BIN_DIR)/mullapudi2016_%) \
 	$(LI2018_TESTS:$(ROOT_DIR)/test/autoschedulers/li2018/%.cpp=$(BIN_DIR)/li2018_%) \
-	$(ADAMS2019_TESTS:$(ROOT_DIR)/test/autoschedulers/adams2019/%.cpp=$(BIN_DIR)/adams2019_%)
+	$(ADAMS2019_TESTS:$(ROOT_DIR)/test/autoschedulers/adams2019/%.cpp=$(BIN_DIR)/adams2019_%) \
+	$(COMMON_AUTOSCHEDULER_TESTS:$(ROOT_DIR)/test/autoschedulers/common/%.cpp=$(BIN_DIR)/common_autoscheduler_%)
 
 clean_generator:
 	rm -rf $(BIN_DIR)/*.generator
@@ -1499,9 +1501,15 @@ $(BIN_DIR)/mullapudi2016_%: $(ROOT_DIR)/test/autoschedulers/mullapudi2016/%.cpp
 $(BIN_DIR)/li2018_%: $(ROOT_DIR)/test/autoschedulers/li2018/%.cpp $(TEST_DEPS)
 	$(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@
 
+$(BIN_DIR)/adams2019_test_function_dag: $(ROOT_DIR)/test/autoschedulers/adams2019/test_function_dag.cpp $(SRC_DIR)/autoschedulers/adams2019/FunctionDAG.cpp $(SRC_DIR)/autoschedulers/common/ASLog.cpp $(TEST_DEPS)
+	$(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< $(SRC_DIR)/autoschedulers/adams2019/FunctionDAG.cpp $(SRC_DIR)/autoschedulers/common/ASLog.cpp -I$(INCLUDE_DIR) -I$(SRC_DIR)/autoschedulers/adams2019 -I$(SRC_DIR)/autoschedulers/common $(TEST_LD_FLAGS) -o $@
+
 $(BIN_DIR)/adams2019_%: $(ROOT_DIR)/test/autoschedulers/adams2019/%.cpp $(TEST_DEPS)
 	$(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) $(TEST_LD_FLAGS) -o $@
 
+$(BIN_DIR)/common_autoscheduler_%: $(ROOT_DIR)/test/autoschedulers/common/%.cpp $(TEST_DEPS)
+	$(CXX) $(TEST_CXX_FLAGS) $(OPTIMIZE_FOR_BUILD_TIME) $< -I$(INCLUDE_DIR) -I$(SRC_DIR)/autoschedulers/common $(TEST_LD_FLAGS) -o $@
+
 # TODO(srj): this doesn't auto-delete, why not?
 .INTERMEDIATE: $(BIN_DIR)/%.generator
 
@@ -2141,6 +2149,18 @@ adams2019_test: $(BIN_DIR)/adams2019_test $(BIN_ADAMS2019) $(SRC_DIR)/autoschedu
 	cd $(TMP_DIR) ; $(CURDIR)/$< $(realpath $(BIN_ADAMS2019)) $(realpath $(SRC_DIR)/autoschedulers/adams2019/baseline.weights)
 	@-echo
 
+adams2019_test_function_dag: $(BIN_DIR)/adams2019_test_function_dag
+	@-mkdir -p $(TMP_DIR)
+	cd $(TMP_DIR) ; $(CURDIR)/$<
+	@-echo
+
+test_common_autoscheduler: $(COMMON_AUTOSCHEDULER_TESTS:$(ROOT_DIR)/test/autoschedulers/common/%.cpp=common_autoscheduler_%)
+
+common_autoscheduler_%: $(BIN_DIR)/common_autoscheduler_%
+	@-mkdir -p $(TMP_DIR)
+	cd $(TMP_DIR) ; $(CURDIR)/$<
+	@-echo
+
 time_compilation_test_%: $(BIN_DIR)/test_%
 	$(TIME_COMPILATION) compile_times_correctness.csv make -f $(THIS_MAKEFILE) $(@:time_compilation_test_%=test_%)
 
@@ -2165,7 +2185,6 @@ TEST_APPS=\
 	local_laplacian \
 	max_filter \
 	nl_means \
-	onnx \
 	resize \
 	resnet_50 \
 	stencil_chain \

apps/onnx/Makefile

@@ -12,10 +12,12 @@ PROTOC := $(shell which protoc)
 
 ifdef PROTOC
 
+PROTOC_PREFIX := $(dir $(PROTOC))..
+PROTOBUF_PKG_CONFIG := PKG_CONFIG_PATH=$(PROTOC_PREFIX)/lib/pkgconfig pkg-config protobuf-lite
+
 CXXFLAGS += -DGOOGLE_PROTOBUF_NO_RTTI -Wno-sign-compare -Wno-unused-but-set-variable
-CXXFLAGS += -I$(dir $(PROTOC))../include
-LDFLAGS += -L$(dir $(PROTOC))../lib
-LDFLAGS += $(shell pkg-config protobuf-lite --libs)
+CXXFLAGS += $(shell $(PROTOBUF_PKG_CONFIG) --cflags)
+LDFLAGS += $(shell $(PROTOBUF_PKG_CONFIG) --libs)
 
 # Copy onnx.proto to $(BIN)
 $(BIN)/%/onnx/onnx.proto:
@@ -27,7 +29,7 @@ $(BIN)/%/onnx/onnx.proto:
 	fi
 
 # protoc generates two files
-$(BIN)/%/onnx/onnx.pb.cc: $(BIN)/%/onnx/onnx.proto
+$(BIN)/%/onnx/onnx.pb.cc: $(BIN)/%/onnx/onnx.proto $(PROTOC)
 	@sed -i -e 's/package onnx;/package onnx;option optimize_for = LITE_RUNTIME;/g' $<
 	@mkdir -p $(@D)
 	$(PROTOC) --cpp_out=$(@D) --proto_path=$(dir $<) $<
@@ -100,15 +102,14 @@ PY_MODEL_EXT = model_cpp$(PY_EXT)
 PYCXXFLAGS = $(PYBIND11_CFLAGS) $(CXXFLAGS) -Wno-deprecated-register
 
 # Python extension for HalideModel
-$(BIN)/%/$(PY_MODEL_EXT): model.cpp $(BIN)/%/oclib.a
+$(BIN)/%/$(PY_MODEL_EXT): model.cpp $(BIN)/%/oclib.a | $(BIN)/%/onnx/onnx_pb.h
 	$(CXX) $(PYCXXFLAGS) -Wall -shared -fPIC -I$(BIN)/$* $^ $(LIBHALIDE_LDFLAGS) -o $@ $(LDFLAGS)
 
-
 model_test: $(BIN)/$(HL_TARGET)/$(PY_MODEL_EXT)
-	PYTHONPATH="$(BIN)/$(HL_TARGET)/:$$PYTHONPATH" $(PYTHON) -m unittest model_test.py -v
+	PYTHONPATH="$(BIN)/$(HL_TARGET)/:$$PYTHONPATH" DYLD_LIBRARY_PATH="$(PROTOC_PREFIX)/lib:$$DYLD_LIBRARY_PATH" $(PYTHON) -m unittest model_test.py -v
 
 halide_as_onnx_backend_test: $(BIN)/$(HL_TARGET)/$(PY_MODEL_EXT)
-	PYTHONPATH="$(BIN)/$(HL_TARGET)/:$$PYTHONPATH" $(PYTHON) -m unittest halide_as_onnx_backend_test.py -v
+	PYTHONPATH="$(BIN)/$(HL_TARGET)/:$$PYTHONPATH" DYLD_LIBRARY_PATH="$(PROTOC_PREFIX)/lib:$$DYLD_LIBRARY_PATH" $(PYTHON) -m unittest halide_as_onnx_backend_test.py -v
 
 # No Protoc
 else