Skip to content

llvm: Use inline variants of memcpy/memset intrinsics when using -fno-builtin #22903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 22, 2025
Merged

llvm: Use inline variants of memcpy/memset intrinsics when using -fno-builtin #22903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
41b179c
llvm.Builder: Update some intrinsic definitions for LLVM 19.
alexrp Feb 22, 2025
21eb757
llvm: Use inline variants of memcpy/memset intrinsics when using -fno…
alexrp Feb 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 64 additions & 10 deletions src/codegen/llvm.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5771,6 +5771,7 @@ pub const FuncGen = struct {
try o.builder.intValue(.i8, 0xaa),
len,
if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
self.ng.ownerModule().no_builtin,
);
const owner_mod = self.ng.ownerModule();
if (owner_mod.valgrind) {
Expand Down Expand Up @@ -5821,6 +5822,7 @@ pub const FuncGen = struct {
try o.builder.intValue(.i8, 0xaa),
len,
.normal,
self.ng.ownerModule().no_builtin,
);
const owner_mod = self.ng.ownerModule();
if (owner_mod.valgrind) {
Expand Down Expand Up @@ -5902,7 +5904,7 @@ pub const FuncGen = struct {
const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);

_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, "");
_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
return if (isByRef(va_list_ty, zcu))
dest_list
else
Expand All @@ -5913,7 +5915,7 @@ pub const FuncGen = struct {
const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
const src_list = try self.resolveInst(un_op);

_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{}, &.{src_list}, "");
_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
return .none;
}

Expand All @@ -5927,7 +5929,7 @@ pub const FuncGen = struct {
const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);

_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, "");
_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
return if (isByRef(va_list_ty, zcu))
dest_list
else
Expand Down Expand Up @@ -9734,6 +9736,7 @@ pub const FuncGen = struct {
if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
len,
if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
self.ng.ownerModule().no_builtin,
);
if (safety and owner_mod.valgrind) {
try self.valgrindMarkUndef(dest_ptr, len);
Expand Down Expand Up @@ -10041,9 +10044,22 @@ pub const FuncGen = struct {
try o.builder.undefValue(.i8);
const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
if (intrinsic_len0_traps) {
try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind);
try self.safeWasmMemset(
dest_ptr,
fill_byte,
len,
dest_ptr_align,
access_kind,
);
} else {
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
_ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
}
const owner_mod = self.ng.ownerModule();
if (safety and owner_mod.valgrind) {
Expand All @@ -10060,9 +10076,22 @@ pub const FuncGen = struct {
const fill_byte = try o.builder.intValue(.i8, byte_val);
const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
if (intrinsic_len0_traps) {
try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind);
try self.safeWasmMemset(
dest_ptr,
fill_byte,
len,
dest_ptr_align,
access_kind,
);
} else {
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
_ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
}
return .none;
}
Expand All @@ -10077,9 +10106,22 @@ pub const FuncGen = struct {
const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);

if (intrinsic_len0_traps) {
try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind);
try self.safeWasmMemset(
dest_ptr,
fill_byte,
len,
dest_ptr_align,
access_kind,
);
} else {
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
_ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
}
return .none;
}
Expand Down Expand Up @@ -10131,6 +10173,7 @@ pub const FuncGen = struct {
elem_abi_align.toLlvm(),
try o.builder.intValue(llvm_usize_ty, elem_abi_size),
access_kind,
self.ng.ownerModule().no_builtin,
);
} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
Expand Down Expand Up @@ -10158,7 +10201,14 @@ pub const FuncGen = struct {
const end_block = try self.wip.block(2, "MemsetTrapEnd");
_ = try self.wip.brCond(cond, memset_block, end_block, .none);
self.wip.cursor = .{ .block = memset_block };
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
_ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
_ = try self.wip.br(end_block);
self.wip.cursor = .{ .block = end_block };
}
Expand Down Expand Up @@ -10200,6 +10250,7 @@ pub const FuncGen = struct {
src_ptr_ty.ptrAlignment(zcu).toLlvm(),
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
_ = try self.wip.br(end_block);
self.wip.cursor = .{ .block = end_block };
Expand All @@ -10213,6 +10264,7 @@ pub const FuncGen = struct {
src_ptr_ty.ptrAlignment(zcu).toLlvm(),
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
return .none;
}
Expand Down Expand Up @@ -11346,6 +11398,7 @@ pub const FuncGen = struct {
ptr_alignment,
try o.builder.intValue(try o.lowerType(Type.usize), size_bytes),
access_kind,
fg.ng.ownerModule().no_builtin,
);
return result_ptr;
}
Expand Down Expand Up @@ -11513,6 +11566,7 @@ pub const FuncGen = struct {
elem_ty.abiAlignment(zcu).toLlvm(),
try o.builder.intValue(try o.lowerType(Type.usize), elem_ty.abiSize(zcu)),
access_kind,
self.ng.ownerModule().no_builtin,
);
}

Expand Down
27 changes: 19 additions & 8 deletions src/codegen/llvm/Builder.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2634,6 +2634,7 @@ pub const Intrinsic = enum {
cos,
pow,
exp,
exp10,
exp2,
ldexp,
frexp,
Expand Down Expand Up @@ -2801,22 +2802,22 @@ pub const Intrinsic = enum {
.va_start = .{
.ret_len = 0,
.params = &.{
.{ .kind = .{ .type = .ptr } },
.{ .kind = .overloaded },
},
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
},
.va_end = .{
.ret_len = 0,
.params = &.{
.{ .kind = .{ .type = .ptr } },
.{ .kind = .overloaded },
},
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
},
.va_copy = .{
.ret_len = 0,
.params = &.{
.{ .kind = .{ .type = .ptr } },
.{ .kind = .{ .type = .ptr } },
.{ .kind = .overloaded },
.{ .kind = .{ .matches = 0 } },
},
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
},
Expand Down Expand Up @@ -2929,7 +2930,7 @@ pub const Intrinsic = enum {
.params = &.{
.{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .writeonly } },
.{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .readonly } },
.{ .kind = .overloaded, .attrs = &.{.immarg} },
.{ .kind = .overloaded },
.{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} },
},
.attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .readwrite } } },
Expand Down Expand Up @@ -2959,7 +2960,7 @@ pub const Intrinsic = enum {
.params = &.{
.{ .kind = .overloaded, .attrs = &.{ .nocapture, .writeonly } },
.{ .kind = .{ .type = .i8 } },
.{ .kind = .overloaded, .attrs = &.{.immarg} },
.{ .kind = .overloaded },
.{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} },
},
.attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .write } } },
Expand Down Expand Up @@ -3022,6 +3023,14 @@ pub const Intrinsic = enum {
},
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } },
},
.exp10 = .{
.ret_len = 1,
.params = &.{
.{ .kind = .overloaded },
.{ .kind = .{ .matches = 0 } },
},
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } },
},
.ldexp = .{
.ret_len = 1,
.params = &.{
Expand Down Expand Up @@ -6093,6 +6102,7 @@ pub const WipFunction = struct {
src_align: Alignment,
len: Value,
kind: MemoryAccessKind,
@"inline": bool,
) Allocator.Error!Instruction.Index {
var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })};
var src_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = src_align })};
Expand All @@ -6104,7 +6114,7 @@ pub const WipFunction = struct {
try self.builder.attrs(&dst_attrs),
try self.builder.attrs(&src_attrs),
}),
.memcpy,
if (@"inline") .@"memcpy.inline" else .memcpy,
&.{ dst.typeOfWip(self), src.typeOfWip(self), len.typeOfWip(self) },
&.{ dst, src, len, switch (kind) {
.normal => Value.false,
Expand All @@ -6122,12 +6132,13 @@ pub const WipFunction = struct {
val: Value,
len: Value,
kind: MemoryAccessKind,
@"inline": bool,
) Allocator.Error!Instruction.Index {
var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })};
const value = try self.callIntrinsic(
.normal,
try self.builder.fnAttrs(&.{ .none, .none, try self.builder.attrs(&dst_attrs) }),
.memset,
if (@"inline") .@"memset.inline" else .memset,
&.{ dst.typeOfWip(self), len.typeOfWip(self) },
&.{ dst, val, len, switch (kind) {
.normal => Value.false,
Expand Down
Loading