radv,nir: add intrinsics for streamout and GS copy shaders

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Qiang Yu <yuq825@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19302>

src/amd/llvm/ac_nir_to_llvm.c

@@ -3619,6 +3619,7 @@ static bool visit_intrinsic(struct ac_nir_context *ctx, nir_intrinsic_instr *ins
    case nir_intrinsic_load_ring_esgs_amd:
    case nir_intrinsic_load_ring_es2gs_offset_amd:
    case nir_intrinsic_load_ring_attr_amd:
+   case nir_intrinsic_load_ring_gsvs_amd:
    case nir_intrinsic_load_lshs_vertex_stride_amd:
    case nir_intrinsic_load_tcs_num_patches_amd:
    case nir_intrinsic_load_hs_out_patch_data_offset_amd:

src/amd/vulkan/radv_nir_lower_abi.c

@@ -35,6 +35,7 @@ typedef struct {
    const struct radv_shader_info *info;
    const struct radv_pipeline_key *pl_key;
    bool use_llvm;
+   uint32_t address32_hi;
 } lower_abi_state;
 
 static nir_ssa_def *
@@ -123,6 +124,12 @@ lower_abi_instr(nir_builder *b, nir_instr *instr, void *state)
 
       replacement = load_ring(b, stage == MESA_SHADER_GEOMETRY ? RING_ESGS_GS : RING_ESGS_VS, s);
       break;
+   case nir_intrinsic_load_ring_gsvs_amd:
+      if (s->use_llvm)
+         break;
+
+      replacement = load_ring(b, RING_GSVS_VS, s);
+      break;
    case nir_intrinsic_load_ring_es2gs_offset_amd:
       replacement = ac_nir_load_arg(b, &s->args->ac, s->args->ac.es2gs_offset);
       break;
@@ -346,6 +353,23 @@ lower_abi_instr(nir_builder *b, nir_instr *instr, void *state)
       /* No-op for RADV. */
       break;
 
+   case nir_intrinsic_load_streamout_config_amd:
+      replacement = ac_nir_load_arg(b, &s->args->ac, s->args->ac.streamout_config);
+      break;
+   case nir_intrinsic_load_streamout_write_index_amd:
+      replacement = ac_nir_load_arg(b, &s->args->ac, s->args->ac.streamout_write_index);
+      break;
+   case nir_intrinsic_load_streamout_buffer_amd: {
+      nir_ssa_def *ptr =
+         nir_pack_64_2x32_split(b, ac_nir_load_arg(b, &s->args->ac, s->args->streamout_buffers),
+                                nir_imm_int(b, s->address32_hi));
+      replacement = nir_load_smem_amd(b, 4, ptr, nir_imm_int(b, nir_intrinsic_base(intrin) * 16));
+      break;
+   }
+   case nir_intrinsic_load_streamout_offset_amd:
+      replacement =
+         ac_nir_load_arg(b, &s->args->ac, s->args->ac.streamout_offset[nir_intrinsic_base(intrin)]);
+      break;
    default:
       progress = false;
       break;
@@ -366,14 +390,15 @@ lower_abi_instr(nir_builder *b, nir_instr *instr, void *state)
 void
 radv_nir_lower_abi(nir_shader *shader, enum amd_gfx_level gfx_level,
                    const struct radv_shader_info *info, const struct radv_shader_args *args,
-                   const struct radv_pipeline_key *pl_key, bool use_llvm)
+                   const struct radv_pipeline_key *pl_key, bool use_llvm, uint32_t address32_hi)
 {
    lower_abi_state state = {
       .gfx_level = gfx_level,
       .info = info,
       .args = args,
       .pl_key = pl_key,
       .use_llvm = use_llvm,
+      .address32_hi = address32_hi,
    };
 
    nir_shader_instructions_pass(shader, lower_abi_instr,

src/amd/vulkan/radv_nir_to_llvm.c

@@ -1335,6 +1335,8 @@ static LLVMValueRef radv_intrinsic_load(struct ac_shader_abi *abi, nir_intrinsic
       return ctx->esgs_ring;
    case nir_intrinsic_load_ring_attr_amd:
       return ctx->attr_ring;
+   case nir_intrinsic_load_ring_gsvs_amd:
+      return ctx->gsvs_ring[0];
    default:
       return NULL;
    }

src/amd/vulkan/radv_pipeline.c

@@ -3895,7 +3895,8 @@ radv_postprocess_nir(struct radv_pipeline *pipeline,
 
    NIR_PASS(_, stage->nir, ac_nir_lower_global_access);
    NIR_PASS_V(stage->nir, radv_nir_lower_abi, gfx_level, &stage->info, &stage->args, pipeline_key,
-              radv_use_llvm_for_stage(device, stage->stage));
+              radv_use_llvm_for_stage(device, stage->stage),
+              device->physical_device->rad_info.address32_hi);
    radv_optimize_nir_algebraic(
       stage->nir, io_to_mem || lowered_ngg || stage->stage == MESA_SHADER_COMPUTE ||
       stage->stage == MESA_SHADER_TASK);

src/amd/vulkan/radv_shader.h

@@ -547,7 +547,8 @@ nir_shader *radv_shader_spirv_to_nir(struct radv_device *device,
 
 void radv_nir_lower_abi(nir_shader *shader, enum amd_gfx_level gfx_level,
                         const struct radv_shader_info *info, const struct radv_shader_args *args,
-                        const struct radv_pipeline_key *pl_key, bool use_llvm);
+                        const struct radv_pipeline_key *pl_key, bool use_llvm,
+                        uint32_t address32_hi);
 
 void radv_init_shader_arenas(struct radv_device *device);
 void radv_destroy_shader_arenas(struct radv_device *device);

src/compiler/nir/nir_divergence_analysis.c

@@ -156,6 +156,10 @@ visit_intrinsic(nir_shader *shader, nir_intrinsic_instr *instr)
    case nir_intrinsic_load_ring_task_payload_amd:
    case nir_intrinsic_load_sample_positions_amd:
    case nir_intrinsic_load_rasterization_samples_amd:
+   case nir_intrinsic_load_ring_gsvs_amd:
+   case nir_intrinsic_load_streamout_config_amd:
+   case nir_intrinsic_load_streamout_write_index_amd:
+   case nir_intrinsic_load_streamout_offset_amd:
    case nir_intrinsic_load_task_ring_entry_amd:
    case nir_intrinsic_load_task_ib_addr:
    case nir_intrinsic_load_task_ib_stride:

src/compiler/nir/nir_intrinsics.py

@@ -1342,6 +1342,16 @@ def store(name, srcs, indices=[], flags=[]):
 # Number of rasterization samples
 system_value("rasterization_samples_amd", 1)
 
+# Descriptor where GS outputs are stored for GS copy shader to read on GFX6-9
+system_value("ring_gsvs_amd", 4)
+
+# Streamout configuration
+system_value("streamout_config_amd", 1)
+# Position to write within the streamout buffers
+system_value("streamout_write_index_amd", 1)
+# Offset to write within a streamout buffer
+system_value("streamout_offset_amd", 1, indices=[BASE])
+
 # AMD merged shader intrinsics
 
 # Whether the current invocation has an input vertex / primitive to process (also known as "ES thread" or "GS thread").