diff --git a/3rdparty/QoLA b/3rdparty/QoLA index 239be99d59..a1f9245e42 160000 --- a/3rdparty/QoLA +++ b/3rdparty/QoLA @@ -1 +1 @@ -Subproject commit 239be99d5906a0c7f7a202b293abb72dace5f283 +Subproject commit a1f9245e4263a172d56e1ec490b3729c0f790b6c diff --git a/3rdparty/ck_jit b/3rdparty/ck_jit index 882083cb84..3cf034d344 160000 --- a/3rdparty/ck_jit +++ b/3rdparty/ck_jit @@ -1 +1 @@ -Subproject commit 882083cb84e7af35cceec403c05d3d1594eaaf1f +Subproject commit 3cf034d3445d7faf5de852840c4419ab720483cf diff --git a/ci/ck_jit_prebuild.txt b/ci/ck_jit_prebuild.txt index 5c06af1cb4..13e927fd51 100644 --- a/ci/ck_jit_prebuild.txt +++ b/ci/ck_jit_prebuild.txt @@ -2,10 +2,12 @@ fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_npad_deterministic_gfx9 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_npad_deterministic_gfx950 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_npad_ndeterministic_gfx9 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_npad_ndeterministic_gfx950 +fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_pd_deterministic_gfx950 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_pd_ndeterministic_gfx9 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_pd_ndeterministic_gfx950 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_ps_ndeterministic_gfx9 fmha_bwd_convert_dq_d128_bf16_b64_batch_o2_ps_ndeterministic_gfx950 +fmha_bwd_convert_dq_d128_bf16_b64_group_o2_psd_deterministic_gfx950 fmha_bwd_convert_dq_d128_bf16_b64_group_o2_ps_deterministic_gfx9 fmha_bwd_convert_dq_d128_bf16_b64_group_o2_ps_deterministic_gfx950 fmha_bwd_convert_dq_d128_bf16_b64_group_o2_ps_ndeterministic_gfx9 @@ -29,6 +31,8 @@ fmha_bwd_convert_dq_d256_bf16_b64_batch_o2_psd_ndeterministic_gfx9 fmha_bwd_convert_dq_d256_bf16_b64_batch_o2_psd_ndeterministic_gfx950 fmha_bwd_convert_dq_d256_bf16_b64_batch_o2_ps_ndeterministic_gfx9 fmha_bwd_convert_dq_d256_bf16_b64_batch_o2_ps_ndeterministic_gfx950 +fmha_bwd_convert_dq_d256_bf16_b64_group_o2_psd_ndeterministic_gfx9 +fmha_bwd_convert_dq_d256_bf16_b64_group_o2_psd_ndeterministic_gfx950 fmha_bwd_convert_dq_d256_bf16_b64_group_o2_ps_ndeterministic_gfx9 fmha_bwd_convert_dq_d256_bf16_b64_group_o2_ps_ndeterministic_gfx950 fmha_bwd_convert_dq_d256_fp16_b64_batch_o2_pd_ndeterministic_gfx9 @@ -49,21 +53,33 @@ fmha_bwd_convert_dq_d64_bf16_b64_batch_o2_npad_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_bf16_b64_batch_o2_pd_deterministic_gfx950 fmha_bwd_convert_dq_d64_bf16_b64_batch_o2_pd_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_bf16_b64_batch_o2_psd_ndeterministic_gfx950 +fmha_bwd_convert_dq_d64_bf16_b64_batch_o2_ps_ndeterministic_gfx9 +fmha_bwd_convert_dq_d64_bf16_b64_batch_o2_ps_ndeterministic_gfx950 +fmha_bwd_convert_dq_d64_bf16_b64_group_o2_psd_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_bf16_b64_group_o2_ps_ndeterministic_gfx9 fmha_bwd_convert_dq_d64_bf16_b64_group_o2_ps_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_fp16_b64_batch_o2_npad_ndeterministic_gfx9 fmha_bwd_convert_dq_d64_fp16_b64_batch_o2_npad_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_fp16_b64_batch_o2_pd_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_fp16_b64_batch_o2_psd_ndeterministic_gfx950 +fmha_bwd_convert_dq_d64_fp16_b64_batch_o2_ps_ndeterministic_gfx9 +fmha_bwd_convert_dq_d64_fp16_b64_batch_o2_ps_ndeterministic_gfx950 fmha_bwd_convert_dq_d64_fp16_b64_group_o2_ps_ndeterministic_gfx9 fmha_bwd_convert_dq_d64_fp16_b64_group_o2_ps_ndeterministic_gfx950 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_alibi_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_alibi_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_mask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_mask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_nmask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_nmask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_mask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_nmask_dropout_wg16_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 @@ -81,11 +97,18 @@ fmha_bwd_d128_bf16_batch_b16x16x128x16x128x16x16x128x128_r1x1x1_r1x1x1_r1x1x1_w1 fmha_bwd_d128_bf16_batch_b16x16x128x16x128x16x16x128x128_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq16_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_alibi_ndbias_mask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_alibi_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_mask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_mask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_nmask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_dbias_nmask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_mask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_mask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_nmask_dropout_wg16_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_bias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_bias_dbias_mask_ndropout_deterministic_trload_gfx950 @@ -102,15 +125,19 @@ fmha_bwd_d128_bf16_batch_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w fmha_bwd_d128_bf16_batch_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_batch_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pdv8_nbias_ndbias_mask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pdv8_nbias_ndbias_nmask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_bf16_group_b16x16x128x16x128x16x16x128x128_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq16_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_trload_gfx950 +fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_group_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_mask_ndropout_deterministic_trload_gfx950 @@ -119,6 +146,7 @@ fmha_bwd_d128_bf16_group_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w fmha_bwd_d128_bf16_group_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_bf16_group_b32x128x128x32x128x32x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_fp16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_deterministic_ntrload_gfx9 +fmha_bwd_d128_fp16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d128_fp16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_ntrload_gfx9 fmha_bwd_d128_fp16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d128_fp16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 @@ -137,6 +165,7 @@ fmha_bwd_d128_fp16_batch_b16x128x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w fmha_bwd_d128_fp16_batch_b16x16x128x16x128x16x16x128x128_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq16_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_fp16_batch_b16x16x128x16x128x16x16x128x128_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq16_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_fp16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_deterministic_trload_gfx950 +fmha_bwd_d128_fp16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_fp16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_deterministic_trload_gfx950 fmha_bwd_d128_fp16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d128_fp16_batch_b16x192x128x16x128x16x32x128x128_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_bias_dbias_mask_dropout_wg16_ndeterministic_trload_gfx950 @@ -185,6 +214,10 @@ fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w1 fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx950 fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx950 +fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx950 +fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 +fmha_bwd_d256_bf16_group_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx950 fmha_bwd_d256_fp16_batch_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d256_fp16_batch_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx950 fmha_bwd_d256_fp16_batch_b16x64x256x16x256x16x32x256x256_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pd8dv8_nbias_ndbias_mask_ndropout_ndeterministic_ntrload_gfx9 @@ -248,6 +281,7 @@ fmha_bwd_d64_bf16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16 fmha_bwd_d64_bf16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pdv8_nbias_ndbias_nmask_dropout_wg16_ndeterministic_ntrload_gfx950 fmha_bwd_d64_bf16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pdv8_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d64_bf16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_pdv8_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx950 +fmha_bwd_d64_bf16_batch_b32x16x64x32x64x32x16x64x64_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq32_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d64_bf16_batch_b32x16x64x32x64x32x16x64x64_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq32_pd8dv8_nbias_ndbias_nmask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d64_bf16_batch_b32x256x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_alibi_ndbias_mask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d64_bf16_batch_b32x256x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_bias_dbias_mask_dropout_wg16_ndeterministic_trload_gfx950 @@ -316,6 +350,7 @@ fmha_bwd_d64_fp16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16 fmha_bwd_d64_fp16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_dropout_wg16_ndeterministic_ntrload_gfx9 fmha_bwd_d64_fp16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx9 fmha_bwd_d64_fp16_batch_b32x128x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x16_o1_maxq0_npad_nbias_ndbias_nmask_ndropout_ndeterministic_ntrload_gfx950 +fmha_bwd_d64_fp16_batch_b32x16x64x32x64x32x16x64x64_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq32_npad_nbias_ndbias_nmask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d64_fp16_batch_b32x16x64x32x64x32x16x64x64_r1x1x1_r1x1x1_r1x1x1_w16x16x32_w16x16x16_o2_maxq32_pd8dv8_nbias_ndbias_nmask_dropout_wg16_ndeterministic_trload_gfx950 fmha_bwd_d64_fp16_batch_b32x256x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_alibi_ndbias_mask_ndropout_ndeterministic_trload_gfx950 fmha_bwd_d64_fp16_batch_b32x256x64x32x64x32x32x64x64_r1x4x1_r4x1x1_r1x4x1_w16x16x32_w16x16x32_o1_maxq0_npad_bias_dbias_mask_dropout_wg16_ndeterministic_trload_gfx950 @@ -374,6 +409,8 @@ fmha_bwd_dot_do_o_d256_bf16_b64_batch_o2_psdv_gfx9 fmha_bwd_dot_do_o_d256_bf16_b64_batch_o2_psdv_gfx950 fmha_bwd_dot_do_o_d256_bf16_b64_batch_o2_ps_gfx9 fmha_bwd_dot_do_o_d256_bf16_b64_batch_o2_ps_gfx950 +fmha_bwd_dot_do_o_d256_bf16_b64_group_o2_psdv_gfx9 +fmha_bwd_dot_do_o_d256_bf16_b64_group_o2_psdv_gfx950 fmha_bwd_dot_do_o_d256_bf16_b64_group_o2_ps_gfx9 fmha_bwd_dot_do_o_d256_bf16_b64_group_o2_ps_gfx950 fmha_bwd_dot_do_o_d256_fp16_b64_batch_o2_pdv_gfx9 @@ -394,6 +431,8 @@ fmha_bwd_dot_do_o_d64_bf16_b64_batch_o2_npad_gfx950 fmha_bwd_dot_do_o_d64_bf16_b64_batch_o2_pdv_gfx9 fmha_bwd_dot_do_o_d64_bf16_b64_batch_o2_pdv_gfx950 fmha_bwd_dot_do_o_d64_bf16_b64_batch_o2_psdv_gfx950 +fmha_bwd_dot_do_o_d64_bf16_b64_batch_o2_ps_gfx9 +fmha_bwd_dot_do_o_d64_bf16_b64_batch_o2_ps_gfx950 fmha_bwd_dot_do_o_d64_bf16_b64_group_o2_psdv_gfx9 fmha_bwd_dot_do_o_d64_bf16_b64_group_o2_psdv_gfx950 fmha_bwd_dot_do_o_d64_bf16_b64_group_o2_ps_gfx9 @@ -402,16 +441,26 @@ fmha_bwd_dot_do_o_d64_fp16_b64_batch_o2_npad_gfx9 fmha_bwd_dot_do_o_d64_fp16_b64_batch_o2_npad_gfx950 fmha_bwd_dot_do_o_d64_fp16_b64_batch_o2_pdv_gfx950 fmha_bwd_dot_do_o_d64_fp16_b64_batch_o2_psdv_gfx950 +fmha_bwd_dot_do_o_d64_fp16_b64_batch_o2_ps_gfx9 +fmha_bwd_dot_do_o_d64_fp16_b64_batch_o2_ps_gfx950 fmha_bwd_dot_do_o_d64_fp16_b64_group_o2_ps_gfx9 fmha_bwd_dot_do_o_d64_fp16_b64_group_o2_ps_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_alibi_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_alibi_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_alibi_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_alibi_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_bias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 @@ -427,8 +476,12 @@ fmha_fwd_d128_bf16_batch_b16x32x64x128x32x128_r1x1x1_r1x1x1_w16x16x32_w16x16x32_ fmha_fwd_d128_bf16_batch_b16x32x64x128x32x128_r1x1x1_r1x1x1_w16x16x32_w16x16x32_qr_async_trload_vr_pddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_trload_nsink_gfx950 fmha_fwd_d128_bf16_batch_b64x128x32x128x32x128_r4x1x1_r4x1x1_w16x16x32_w16x16x16_qr_async_vr_psddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_batch_b64x128x32x128x32x128_r4x1x1_r4x1x1_w16x16x32_w16x16x16_qr_async_vr_psddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 +fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d128_bf16_group_b128x128x32x128x32x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d128_bf16_group_b128x64x32x128x16x128_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_trload_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_trload_nsink_gfx950 @@ -493,27 +546,27 @@ fmha_fwd_d192_fp16_batch_b128x128x32x128x32x192_r4x1x1_r4x1x1_w32x32x16_w32x32x1 fmha_fwd_d192_fp16_batch_b128x128x32x192x32x192_r4x1x1_r4x1x1_w32x32x16_w32x32x16_o1_qr_async_vr_psddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d192_fp16_batch_b128x128x32x192x32x192_r4x1x1_r4x1x1_w32x32x16_w32x32x16_o1_qr_async_vr_psddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_group_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_group_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_bf16_group_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_bf16_group_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_group_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_bf16_group_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_fp16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_fp16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_fp16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_fp16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_fp16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_fp16_batch_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_fp16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_fp16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_fp16_batch_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d256_fp16_group_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 -fmha_fwd_d256_fp16_group_b128x128x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d256_fp16_group_b128x64x32x256x32x256_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_pssk_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d32_bf16_batch_b128x64x16x32x32x32_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d32_bf16_batch_b128x64x16x32x32x32_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d32_bf16_batch_b128x64x16x32x32x32_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psddv_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 @@ -561,6 +614,7 @@ fmha_fwd_d64_bf16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr fmha_fwd_d64_bf16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_bias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d64_bf16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_bias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d64_bf16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_psskddv_nlogits_bias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d64_bf16_batch_b16x32x64x64x32x64_r1x1x1_r1x1x1_w16x16x32_w16x16x32_qr_async_trload_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_trload_nsink_gfx950 fmha_fwd_d64_bf16_group_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d64_bf16_group_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d64_bf16_group_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 @@ -587,6 +641,8 @@ fmha_fwd_d64_fp16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr fmha_fwd_d64_fp16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d64_fp16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d64_fp16_batch_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_vr_npad_nlogits_bias_nmask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx950 +fmha_fwd_d64_fp16_batch_b16x32x64x64x32x64_r1x1x1_r1x1x1_w16x16x32_w16x16x32_qr_async_trload_vr_npad_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_trload_nsink_gfx950 +fmha_fwd_d64_fp16_batch_b16x32x64x64x32x64_r1x1x1_r1x1x1_w16x16x32_w16x16x32_qr_async_trload_vr_npad_nlogits_nbias_nmask_lse_ndropout_nskip_nqscale_trload_nsink_gfx950 fmha_fwd_d64_fp16_group_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx9 fmha_fwd_d64_fp16_group_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_dropout_nskip_nqscale_ntrload_nsink_gfx950 fmha_fwd_d64_fp16_group_b128x64x32x64x32x64_r4x1x1_r4x1x1_w32x32x16_w32x32x16_qr_async_vr_psskddv_nlogits_nbias_mask_lse_ndropout_nskip_nqscale_ntrload_nsink_gfx9 diff --git a/transformer_engine/common/ck_fused_attn/CMakeLists.txt b/transformer_engine/common/ck_fused_attn/CMakeLists.txt index 964ff6513c..2e4c02658d 100644 --- a/transformer_engine/common/ck_fused_attn/CMakeLists.txt +++ b/transformer_engine/common/ck_fused_attn/CMakeLists.txt @@ -222,7 +222,7 @@ add_library(ck_fused_attn SHARED ${ck_fused_attn_SOURCES}) set(CK_FUSED_ATTN_COMPILE_OPTIONS) list(APPEND CK_FUSED_ATTN_COMPILE_OPTIONS -DCK_TILE_FLOAT_TO_BFLOAT16_DEFAULT=${CK_FUSED_ATTN_FLOAT_TO_BFLOAT16_DEFAULT} - -DENABLE_CK=1 -DFAV_NATIVE_ON=1) + -DENABLE_CK=1 -DFA_WITH_NATIVE_SPLITKV=1) # Public QoLA headers ship alongside the .so libs in ${__AITER_MHA_PATH}/../include # (emitted by qola.cli build, or copied from the QoLA build dir above for the diff --git a/transformer_engine/common/ck_fused_attn/qola_manifest.toml b/transformer_engine/common/ck_fused_attn/qola_manifest.toml index f2dc42a1d0..a689234445 100644 --- a/transformer_engine/common/ck_fused_attn/qola_manifest.toml +++ b/transformer_engine/common/ck_fused_attn/qola_manifest.toml @@ -1,7 +1,7 @@ [qola] -aiter_commit = "7a24cd87525834fa9aeaa021e6ee80ab7de028f9" # pinned AITER submodule commit +aiter_commit = "a50a62aaa292427e00190dd7ba58e39a6db4e61e" # pinned AITER submodule commit namespace = "te" -rocm_versions = ["7.2"] +rocm_versions = ["7.14"] [build] architectures = ["gfx950", "gfx942"] @@ -12,6 +12,7 @@ mode = "cpp_itfs" receipt = 700 drop_srcs = ["mha_fwd_split.cu", "mha_fwd_batch_prefill.cu"] drop_directions = ["fwd_splitkv", "batch_prefill"] +flags_extra_cc = ["'-DFA_WITH_NATIVE_SPLITKV=1'"] [[modules]] name = "libmha_bwd" diff --git a/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp b/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp index e73619127b..e6700147bf 100644 --- a/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp +++ b/transformer_engine/common/ck_fused_attn/src/ck_fused_attn_fwd.cpp @@ -267,7 +267,7 @@ hipError_t ck_attn_fwd(const CKAttnFwdArgs& args, hipStream_t stream){ } int ck_attn_fwd_num_splits(const CKAttnFwdArgs& args){ -#if FAV_NATIVE_ON +#if FA_WITH_NATIVE_SPLITKV aiter::mha_fwd_args fmha_args = build_fwd_fmha_args(args); return QOLA_NS(mha_fwd_calculate_num_splits)(fmha_args); #else @@ -276,7 +276,7 @@ int ck_attn_fwd_num_splits(const CKAttnFwdArgs& args){ } size_t ck_attn_fwd_workspace_size(const CKAttnFwdArgs& args){ -#if FAV_NATIVE_ON +#if FA_WITH_NATIVE_SPLITKV aiter::mha_fwd_args fmha_args = build_fwd_fmha_args(args); return QOLA_NS(mha_fwd_workspace_size)(fmha_args); #else