diff --git a/src/compiler/nir/nir_lower_atomics.c b/src/compiler/nir/nir_lower_atomics.c index 778e6ca8b2f..8e409fa930a 100644 --- a/src/compiler/nir/nir_lower_atomics.c +++ b/src/compiler/nir/nir_lower_atomics.c @@ -44,14 +44,33 @@ */ static nir_def * -build_atomic(nir_builder *b, nir_intrinsic_instr *intr, bool ssbo) +build_atomic(nir_builder *b, nir_intrinsic_instr *intr) { - nir_def *load = ssbo ? nir_load_ssbo(b, 1, intr->def.bit_size, intr->src[0].ssa, - intr->src[1].ssa, - .align_mul = intr->def.bit_size / 8, - .align_offset = 0) - : nir_load_global(b, intr->src[0].ssa, 8, 1, intr->def.bit_size); - nir_def *data = ssbo ? intr->src[2].ssa : intr->src[1].ssa; + nir_def *load; + switch(intr->intrinsic) { + case nir_intrinsic_ssbo_atomic: + load = nir_load_ssbo(b, 1, intr->def.bit_size, intr->src[0].ssa, + intr->src[1].ssa, + .align_mul = intr->def.bit_size / 8, + .align_offset = 0); + break; + case nir_intrinsic_shared_atomic: + load = nir_load_shared(b, 1, intr->def.bit_size, + intr->src[0].ssa, + .align_mul = intr->def.bit_size / 8, + .align_offset = 0); + break; + case nir_intrinsic_global_atomic: + load = nir_load_global(b, intr->src[0].ssa, + intr->def.bit_size / 8, + 1, intr->def.bit_size); + break; + default: + unreachable("unsupported atomic type"); + } + + nir_def *data = intr->intrinsic == nir_intrinsic_ssbo_atomic ? + intr->src[2].ssa : intr->src[1].ssa; nir_loop *loop = nir_push_loop(b); nir_def *xchg; { @@ -64,14 +83,28 @@ build_atomic(nir_builder *b, nir_intrinsic_instr *intr, bool ssbo) nir_alu_instr* op = nir_instr_as_alu(expected->parent_instr); op->exact = true; op->fp_fast_math = 0; - if (ssbo) { - xchg = nir_ssbo_atomic_swap(b, intr->def.bit_size, intr->src[0].ssa, intr->src[1].ssa, + switch(intr->intrinsic) { + case nir_intrinsic_ssbo_atomic: + xchg = nir_ssbo_atomic_swap(b, intr->def.bit_size, + intr->src[0].ssa, + intr->src[1].ssa, before, expected, .atomic_op = nir_atomic_op_cmpxchg); - } else { - xchg = - nir_global_atomic_swap(b, intr->def.bit_size, intr->src[0].ssa, before, expected, - .atomic_op = nir_atomic_op_cmpxchg); + break; + case nir_intrinsic_shared_atomic: + xchg = nir_shared_atomic_swap(b,intr->def.bit_size, + intr->src[0].ssa, + before, expected, + .atomic_op = nir_atomic_op_cmpxchg); + break; + case nir_intrinsic_global_atomic: + xchg = nir_global_atomic_swap(b, intr->def.bit_size, + intr->src[0].ssa, + before, expected, + .atomic_op = nir_atomic_op_cmpxchg); + break; + default: + unreachable("unsupported atomic type"); } nir_break_if(b, nir_ieq(b, xchg, before)); nir_phi_instr_add_src(phi, nir_loop_last_block(loop), xchg); @@ -89,6 +122,7 @@ lower_atomics(struct nir_builder *b, nir_intrinsic_instr *intr, nir_instr_filter_cb supported_cb = supported; if (intr->intrinsic != nir_intrinsic_ssbo_atomic && + intr->intrinsic != nir_intrinsic_shared_atomic && intr->intrinsic != nir_intrinsic_global_atomic) return false; if (supported_cb(&intr->instr, NULL)) @@ -106,8 +140,7 @@ lower_atomics(struct nir_builder *b, nir_intrinsic_instr *intr, case nir_atomic_op_fmin: case nir_atomic_op_fmax: case nir_atomic_op_iadd: { - nir_def_replace(&intr->def, build_atomic( - b, intr, intr->intrinsic == nir_intrinsic_ssbo_atomic)); + nir_def_replace(&intr->def, build_atomic(b, intr)); return true; } case nir_atomic_op_cmpxchg: