dsg/mesa
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nir: support shared atomics in nir_lower_atomics

Browse source 
Add support to rewrite shared atomics into compare-and-swap loops,
previously the nir_lower_atomics pass only supported global and ssbo
atomics.

Only freedreno irc3 reuses nir_lower_atomics, this change does not
impact their usage since they do not support shared atomics.

Signed-off-by: Lorenzo Rossi <snowycoder@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/33572>
This commit is contained in:
Lorenzo Rossi 2025-02-17 00:35:46 +01:00 • committed by Marge Bot
parent 15544ed858
commit 26079c1a93
1 changed files with 48 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

63
src/compiler/nir/nir_lower_atomics.c
View file

@ -44,14 +44,33 @@
*/
static nir_def *
build_atomic(nir_builder *b, nir_intrinsic_instr *intr, bool ssbo)
build_atomic(nir_builder *b, nir_intrinsic_instr *intr)
{
nir_def *load = ssbo ? nir_load_ssbo(b, 1, intr->def.bit_size, intr->src[0].ssa,
intr->src[1].ssa,
.align_mul = intr->def.bit_size / 8,
.align_offset = 0)
: nir_load_global(b, intr->src[0].ssa, 8, 1, intr->def.bit_size);
nir_def *data = ssbo ? intr->src[2].ssa : intr->src[1].ssa;
nir_def *load;
switch(intr->intrinsic) {
case nir_intrinsic_ssbo_atomic:
load = nir_load_ssbo(b, 1, intr->def.bit_size, intr->src[0].ssa,
intr->src[1].ssa,
.align_mul = intr->def.bit_size / 8,
.align_offset = 0);
break;
case nir_intrinsic_shared_atomic:
load = nir_load_shared(b, 1, intr->def.bit_size,
intr->src[0].ssa,
.align_mul = intr->def.bit_size / 8,
.align_offset = 0);
break;
case nir_intrinsic_global_atomic:
load = nir_load_global(b, intr->src[0].ssa,
intr->def.bit_size / 8,
1, intr->def.bit_size);
break;
default:
unreachable("unsupported atomic type");
}
nir_def *data = intr->intrinsic == nir_intrinsic_ssbo_atomic ?
intr->src[2].ssa : intr->src[1].ssa;
nir_loop *loop = nir_push_loop(b);
nir_def *xchg;
{
@ -64,14 +83,28 @@ build_atomic(nir_builder *b, nir_intrinsic_instr *intr, bool ssbo)
nir_alu_instr* op = nir_instr_as_alu(expected->parent_instr);
op->exact = true;
op->fp_fast_math = 0;
if (ssbo) {
xchg = nir_ssbo_atomic_swap(b, intr->def.bit_size, intr->src[0].ssa, intr->src[1].ssa,
switch(intr->intrinsic) {
case nir_intrinsic_ssbo_atomic:
xchg = nir_ssbo_atomic_swap(b, intr->def.bit_size,
intr->src[0].ssa,
intr->src[1].ssa,
before, expected,
.atomic_op = nir_atomic_op_cmpxchg);
} else {
xchg =
nir_global_atomic_swap(b, intr->def.bit_size, intr->src[0].ssa, before, expected,
.atomic_op = nir_atomic_op_cmpxchg);
break;
case nir_intrinsic_shared_atomic:
xchg = nir_shared_atomic_swap(b,intr->def.bit_size,
intr->src[0].ssa,
before, expected,
.atomic_op = nir_atomic_op_cmpxchg);
break;
case nir_intrinsic_global_atomic:
xchg = nir_global_atomic_swap(b, intr->def.bit_size,
intr->src[0].ssa,
before, expected,
.atomic_op = nir_atomic_op_cmpxchg);
break;
default:
unreachable("unsupported atomic type");
}
nir_break_if(b, nir_ieq(b, xchg, before));
nir_phi_instr_add_src(phi, nir_loop_last_block(loop), xchg);
@ -89,6 +122,7 @@ lower_atomics(struct nir_builder *b, nir_intrinsic_instr *intr,
nir_instr_filter_cb supported_cb = supported;
if (intr->intrinsic != nir_intrinsic_ssbo_atomic &&
intr->intrinsic != nir_intrinsic_shared_atomic &&
intr->intrinsic != nir_intrinsic_global_atomic)
return false;
if (supported_cb(&intr->instr, NULL))
@ -106,8 +140,7 @@ lower_atomics(struct nir_builder *b, nir_intrinsic_instr *intr,
case nir_atomic_op_fmin:
case nir_atomic_op_fmax:
case nir_atomic_op_iadd: {
nir_def_replace(&intr->def, build_atomic(
b, intr, intr->intrinsic == nir_intrinsic_ssbo_atomic));
nir_def_replace(&intr->def, build_atomic(b, intr));
return true;
}
case nir_atomic_op_cmpxchg: