@@ -1515,6 +1515,17 @@ void CodeGen_LLVM::visit(const Reinterpret *op) {
15151515 llvm::Type *llvm_dst_fixed = get_vector_type (llvm_type_of (dst.element_of ()), dst.lanes (), VectorTypeConstraint::Fixed);
15161516 value = builder->CreateBitOrPointerCast (value, llvm_dst_fixed);
15171517 value = fixed_to_scalable_vector_type (value);
1518+ } else if (isa<FixedVectorType>(value->getType ()) && isa<ScalableVectorType>(llvm_dst)) {
1519+ // Cannot bitcast/ptrtoint directly between fixed and scalable vectors.
1520+ // First cast to a fixed vector of the destination element type, then convert to scalable.
1521+ llvm::Type *llvm_dst_fixed = get_vector_type (llvm_dst->getScalarType (), dst.lanes (), VectorTypeConstraint::Fixed);
1522+ value = builder->CreateBitOrPointerCast (value, llvm_dst_fixed);
1523+ value = fixed_to_scalable_vector_type (value);
1524+ } else if (isa<ScalableVectorType>(value->getType ()) && isa<FixedVectorType>(llvm_dst)) {
1525+ // Cannot bitcast/ptrtoint directly between scalable and fixed vectors.
1526+ // First convert to a fixed vector of the source element type, then cast.
1527+ value = scalable_to_fixed_vector_type (value);
1528+ value = builder->CreateBitOrPointerCast (value, llvm_dst);
15181529 } else {
15191530 // Our `Reinterpret` expr directly maps to LLVM IR bitcast/ptrtoint/inttoptr
15201531 // instructions with no additional handling required:
@@ -4338,10 +4349,12 @@ void CodeGen_LLVM::codegen_vector_reduce(const VectorReduce *op, const Expr &ini
43384349 const int input_lanes = val.type ().lanes ();
43394350 const int input_bytes = input_lanes * val.type ().bytes ();
43404351 const int vscale = std::max (effective_vscale, 1 );
4352+ // LLVM added VECREDUCE_MUL/FMUL lowering for SVE in LLVM 22.
4353+ const bool mul_ok = LLVM_VERSION >= 220 || effective_vscale == 0 ;
43414354 const bool llvm_has_intrinsic =
43424355 // Must be one of these ops
43434356 ((op->op == VectorReduce::Add ||
4344- op->op == VectorReduce::Mul ||
4357+ ( op->op == VectorReduce::Mul && mul_ok) ||
43454358 op->op == VectorReduce::Min ||
43464359 op->op == VectorReduce::Max) &&
43474360 (use_llvm_vp_intrinsics ||
@@ -4950,6 +4963,13 @@ Value *CodeGen_LLVM::slice_vector(Value *vec, int start, int size) {
49504963 // otherwise.
49514964 llvm::Type *scalar_type = vec->getType ()->getScalarType ();
49524965
4966+ if (scalar_type->isIntegerTy (1 )) {
4967+ auto *result_type = cast<VectorType>(get_vector_type (scalar_type, size / effective_vscale, VectorTypeConstraint::VScale));
4968+ return handle_bool_as_i8 (vec, result_type, [&](Value *v) {
4969+ return slice_vector (v, start, size);
4970+ });
4971+ }
4972+
49534973 int intermediate_lanes = std::min (size, vec_lanes - start);
49544974 llvm::Type *intermediate_type = get_vector_type (scalar_type, intermediate_lanes, VectorTypeConstraint::Fixed);
49554975
@@ -5241,6 +5261,18 @@ llvm::Value *CodeGen_LLVM::match_vector_type_scalable(llvm::Value *value, llvm::
52415261 return match_vector_type_scalable (value, guide->getType ());
52425262}
52435263
5264+ llvm::Value *CodeGen_LLVM::handle_bool_as_i8 (llvm::Value *arg, llvm::VectorType *result_i1_type,
5265+ const std::function<llvm::Value *(llvm::Value *)> &fn) {
5266+ auto *arg_vty = cast<llvm::VectorType>(arg->getType ());
5267+ bool scalable = isa<llvm::ScalableVectorType>(arg_vty);
5268+ int min_elts = scalable ? cast<llvm::ScalableVectorType>(arg_vty)->getMinNumElements () : cast<llvm::FixedVectorType>(arg_vty)->getNumElements ();
5269+ auto constraint = scalable ? VectorTypeConstraint::VScale : VectorTypeConstraint::Fixed;
5270+ llvm::Type *arg_i8 = get_vector_type (i8_t , min_elts, constraint);
5271+ llvm::Value *widened = builder->CreateZExt (arg, arg_i8);
5272+ llvm::Value *result = fn (widened);
5273+ return builder->CreateTrunc (result, result_i1_type);
5274+ }
5275+
52445276llvm::Value *CodeGen_LLVM::convert_fixed_or_scalable_vector_type (llvm::Value *arg,
52455277 llvm::Type *desired_type) {
52465278 llvm::Type *arg_type = arg->getType ();
@@ -5250,6 +5282,18 @@ llvm::Value *CodeGen_LLVM::convert_fixed_or_scalable_vector_type(llvm::Value *ar
52505282 }
52515283
52525284 internal_assert (arg_type->getScalarType () == desired_type->getScalarType ());
5285+
5286+ if (arg_type->isVectorTy () && desired_type->isVectorTy () &&
5287+ arg_type->getScalarType ()->isIntegerTy (1 )) {
5288+ bool dst_scalable = isa<llvm::ScalableVectorType>(desired_type);
5289+ int dst_elts = get_vector_num_elements (desired_type);
5290+ llvm::Type *dst_i8 = get_vector_type (i8_t , dst_scalable ? dst_elts / effective_vscale : dst_elts,
5291+ dst_scalable ? VectorTypeConstraint::VScale : VectorTypeConstraint::Fixed);
5292+ return handle_bool_as_i8 (arg, cast<VectorType>(desired_type), [&](Value *v) {
5293+ return convert_fixed_or_scalable_vector_type (v, dst_i8);
5294+ });
5295+ }
5296+
52535297 if (!arg_type->isVectorTy ()) {
52545298 arg = create_broadcast (arg, 1 );
52555299 arg_type = arg->getType ();
@@ -5331,6 +5375,12 @@ llvm::Value *CodeGen_LLVM::fixed_to_scalable_vector_type(llvm::Value *fixed_arg)
53315375 internal_assert (fixed_type->getElementType () == scalable_type->getElementType ());
53325376 internal_assert (lanes == (scalable_type->getMinNumElements () * effective_vscale));
53335377
5378+ if (fixed_type->getElementType ()->isIntegerTy (1 )) {
5379+ return handle_bool_as_i8 (fixed_arg, scalable_type, [&](Value *v) {
5380+ return fixed_to_scalable_vector_type (v);
5381+ });
5382+ }
5383+
53345384 // E.g. <vscale x 2 x i64> llvm.vector.insert.nxv2i64.v4i64(<vscale x 2 x i64>, <4 x i64>, i64)
53355385 const char *type_designator;
53365386 if (fixed_type->getElementType ()->isIntegerTy ()) {
@@ -5348,7 +5398,7 @@ llvm::Value *CodeGen_LLVM::fixed_to_scalable_vector_type(llvm::Value *fixed_arg)
53485398
53495399 std::vector<llvm::Value *> args;
53505400 args.push_back (result_vec);
5351- args.push_back (value );
5401+ args.push_back (fixed_arg );
53525402 args.push_back (ConstantInt::get (i64_t , 0 ));
53535403
53545404 return simple_call_intrin (intrin, args, scalable_type);
@@ -5367,6 +5417,12 @@ llvm::Value *CodeGen_LLVM::scalable_to_fixed_vector_type(llvm::Value *scalable_a
53675417 internal_assert (fixed_type->getElementType () == scalable_type->getElementType ());
53685418 internal_assert (fixed_type->getNumElements () == (scalable_type->getMinNumElements () * effective_vscale));
53695419
5420+ if (scalable_type->getElementType ()->isIntegerTy (1 )) {
5421+ return handle_bool_as_i8 (scalable_arg, fixed_type, [&](Value *v) {
5422+ return scalable_to_fixed_vector_type (v);
5423+ });
5424+ }
5425+
53705426 // E.g. <64 x i8> @llvm.vector.extract.v64i8.nxv8i8(<vscale x 8 x i8> %vresult, i64 0)
53715427 const char *type_designator;
53725428 if (scalable_type->getElementType ()->isIntegerTy ()) {
0 commit comments