Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 23c1f86

Browse files
crcrparcrcrpar
authored
release gil (#1816)
Signed-off-by: Masaki Kozuki <[email protected]>
1 parent f8e60c4 commit 23c1f86

33 files changed

Lines changed: 200 additions & 180 deletions

‎apex/contrib/csrc/bottleneck/bottleneck.cpp‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4048,26 +4048,26 @@ void bottleneck_backward_rest(bool explicit_nhwc, int stride_1X1, std::vector<at
40484048
}
40494049

40504050
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
4051-
m.def("forward", &bottleneck_forward, "Bottleneck block forward");
4052-
m.def("backward", &bottleneck_backward, "Bottleneck block backward");
4053-
m.def("forward_init", &bottleneck_forward_init, "Bottleneck block init");
4054-
m.def("forward_out1", &bottleneck_forward_out1, "Bottleneck block forward");
4055-
m.def("forward_out2", &bottleneck_forward_out2, "Bottleneck block forward");
4056-
m.def("forward_out2_mask", &bottleneck_forward_out2_mask, "Bottleneck block forward");
4057-
m.def("forward_out2_halo", &bottleneck_forward_out2_halo, "Bottleneck block forward");
4058-
m.def("forward_out2_halo_corr", &bottleneck_forward_out2_halo_corr, "Bottleneck block forward");
4059-
m.def("forward_out2_pad", &bottleneck_forward_out2_pad, "Bottleneck block forward");
4060-
m.def("forward_rest", &bottleneck_forward_rest, "Bottleneck block forward");
4061-
m.def("backward_init", &bottleneck_backward_init, "Bottleneck block backward init");
4062-
m.def("backward_grad_out2", &bottleneck_backward_grad_out2, "Bottleneck block backward");
4063-
m.def("backward_grad_out1", &bottleneck_backward_grad_out1, "Bottleneck block backward");
4064-
m.def("backward_grad_out1_mask", &bottleneck_backward_grad_out1_mask, "Bottleneck block backward");
4065-
m.def("backward_grad_out1_halo", &bottleneck_backward_grad_out1_halo, "Bottleneck block backward");
4066-
m.def("backward_grad_out1_halo_corr", &bottleneck_backward_grad_out1_halo_corr, "Bottleneck block backward");
4067-
m.def("backward_wgrad2_pad", &bottleneck_backward_wgrad2_pad, "Bottleneck block backward");
4068-
m.def("backward_wgrad2", &bottleneck_backward_wgrad2, "Bottleneck block backward");
4069-
m.def("backward_wgrad2_halo", &bottleneck_backward_wgrad2_halo, "Bottleneck block backward");
4070-
m.def("backward_wgrad3", &bottleneck_backward_wgrad3, "Bottleneck block backward");
4071-
m.def("backward_wgrad1", &bottleneck_backward_wgrad1, "Bottleneck block backward");
4072-
m.def("backward_rest", &bottleneck_backward_rest, "Bottleneck block backward");
4051+
m.def("forward", &bottleneck_forward, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4052+
m.def("backward", &bottleneck_backward, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4053+
m.def("forward_init", &bottleneck_forward_init, "Bottleneck block init", py::call_guard<py::gil_scoped_release>());
4054+
m.def("forward_out1", &bottleneck_forward_out1, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4055+
m.def("forward_out2", &bottleneck_forward_out2, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4056+
m.def("forward_out2_mask", &bottleneck_forward_out2_mask, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4057+
m.def("forward_out2_halo", &bottleneck_forward_out2_halo, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4058+
m.def("forward_out2_halo_corr", &bottleneck_forward_out2_halo_corr, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4059+
m.def("forward_out2_pad", &bottleneck_forward_out2_pad, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4060+
m.def("forward_rest", &bottleneck_forward_rest, "Bottleneck block forward", py::call_guard<py::gil_scoped_release>());
4061+
m.def("backward_init", &bottleneck_backward_init, "Bottleneck block backward init", py::call_guard<py::gil_scoped_release>());
4062+
m.def("backward_grad_out2", &bottleneck_backward_grad_out2, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4063+
m.def("backward_grad_out1", &bottleneck_backward_grad_out1, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4064+
m.def("backward_grad_out1_mask", &bottleneck_backward_grad_out1_mask, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4065+
m.def("backward_grad_out1_halo", &bottleneck_backward_grad_out1_halo, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4066+
m.def("backward_grad_out1_halo_corr", &bottleneck_backward_grad_out1_halo_corr, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4067+
m.def("backward_wgrad2_pad", &bottleneck_backward_wgrad2_pad, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4068+
m.def("backward_wgrad2", &bottleneck_backward_wgrad2, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4069+
m.def("backward_wgrad2_halo", &bottleneck_backward_wgrad2_halo, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4070+
m.def("backward_wgrad3", &bottleneck_backward_wgrad3, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4071+
m.def("backward_wgrad1", &bottleneck_backward_wgrad1, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
4072+
m.def("backward_rest", &bottleneck_backward_rest, "Bottleneck block backward", py::call_guard<py::gil_scoped_release>());
40734073
}

‎apex/contrib/csrc/conv_bias_relu/conv_bias_relu.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2142,12 +2142,12 @@ std::vector<at::Tensor> conv_bias_backward(std::vector<at::Tensor> inputs, int64
21422142
}
21432143

21442144
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
2145-
m.def("forward", &conv_bias_relu_forward, "Fused Conv-Bias-ReLU forward");
2146-
m.def("backward", &conv_bias_relu_backward, "Fused Conv-Bias-ReLU backward");
2147-
m.def("forward_no_relu", &conv_bias_forward, "Fused Conv-Bias forward");
2148-
m.def("backward_no_relu", &conv_bias_backward, "Fused Conv-Bias backward");
2149-
m.def("forward_mask", &conv_bias_mask_relu_forward, "Fused Conv-Bias-Mask-ReLU forward");
2150-
m.def("forward_cscale_cbias_relu", &conv_cscale_cbias_relu_forward, "Fused Conv-(const)Scale-(const)Bias-ReLU");
2151-
m.def("backward_cscale_cbias_relu", &conv_cscale_cbias_relu_backward, "Fused Conv-(const)Scale-(const)Bias-ReLU backward");
2145+
m.def("forward", &conv_bias_relu_forward, "Fused Conv-Bias-ReLU forward", py::call_guard<py::gil_scoped_release>());
2146+
m.def("backward", &conv_bias_relu_backward, "Fused Conv-Bias-ReLU backward", py::call_guard<py::gil_scoped_release>());
2147+
m.def("forward_no_relu", &conv_bias_forward, "Fused Conv-Bias forward", py::call_guard<py::gil_scoped_release>());
2148+
m.def("backward_no_relu", &conv_bias_backward, "Fused Conv-Bias backward", py::call_guard<py::gil_scoped_release>());
2149+
m.def("forward_mask", &conv_bias_mask_relu_forward, "Fused Conv-Bias-Mask-ReLU forward", py::call_guard<py::gil_scoped_release>());
2150+
m.def("forward_cscale_cbias_relu", &conv_cscale_cbias_relu_forward, "Fused Conv-(const)Scale-(const)Bias-ReLU", py::call_guard<py::gil_scoped_release>());
2151+
m.def("backward_cscale_cbias_relu", &conv_cscale_cbias_relu_backward, "Fused Conv-(const)Scale-(const)Bias-ReLU backward", py::call_guard<py::gil_scoped_release>());
21522152
}
21532153

‎apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,6 @@ std::vector<at::Tensor> gbn_backward(
158158

159159

160160
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
161-
m.def("forward", &gbn_forward, "Group batch norm forward");
162-
m.def("backward", &gbn_backward, "Group batch backward");
161+
m.def("forward", &gbn_forward, "Group batch norm forward", py::call_guard<py::gil_scoped_release>());
162+
m.def("backward", &gbn_backward, "Group batch backward", py::call_guard<py::gil_scoped_release>());
163163
}

‎apex/contrib/csrc/fmha/fmha_api.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,7 @@ std::vector<at::Tensor> mha_bwd_nl(const at::Tensor &dout, // total x num
359359

360360
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
361361
m.doc() = "Fused Multi-head Self-attention for BERT";
362-
m.def("fwd", &mha_fwd, "Forward pass");
363-
m.def("bwd", &mha_bwd, "Backward pass");
364-
m.def("bwd_nl", &mha_bwd_nl, "Backward pass (small-batch)");
362+
m.def("fwd", &mha_fwd, "Forward pass", py::call_guard<py::gil_scoped_release>());
363+
m.def("bwd", &mha_bwd, "Backward pass", py::call_guard<py::gil_scoped_release>());
364+
m.def("bwd_nl", &mha_bwd_nl, "Backward pass (small-batch)", py::call_guard<py::gil_scoped_release>());
365365
}

‎apex/contrib/csrc/focal_loss/focal_loss_cuda.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,9 @@ at::Tensor focal_loss_backward(
6464

6565
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6666
m.def("forward", &focal_loss_forward,
67-
"Focal loss calculation forward (CUDA)");
67+
"Focal loss calculation forward (CUDA)",
68+
py::call_guard<py::gil_scoped_release>());
6869
m.def("backward", &focal_loss_backward,
69-
"Focal loss calculation backward (CUDA)");
70+
"Focal loss calculation backward (CUDA)",
71+
py::call_guard<py::gil_scoped_release>());
7072
}

‎apex/contrib/csrc/group_norm/group_norm_nhwc_op.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
305305
.value("OnePass", OnePass)
306306
.value("TwoPass", TwoPass)
307307
.export_values();
308-
m.def("forward", &group_norm_fwd, "NHWC group norm forward");
309-
m.def("backward", &group_norm_bwd, "NHWC group norm backward");
308+
m.def("forward", &group_norm_fwd, "NHWC group norm forward", py::call_guard<py::gil_scoped_release>());
309+
m.def("backward", &group_norm_bwd, "NHWC group norm backward", py::call_guard<py::gil_scoped_release>());
310310
}

‎apex/contrib/csrc/groupbn/interface.cpp‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -153,23 +153,23 @@ int nhwc_bn_addrelu_bwd_occupancy();
153153

154154
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
155155

156-
m.def("get_buffer_size", &get_buffer_size, "get_buffer_size");
157-
m.def("get_data_ptr", &get_data_ptr, "get_data_ptr");
158-
m.def("get_remote_data_ptr", &get_remote_data_ptr, "get_remote_data_ptr");
159-
m.def("close_remote_data", &close_remote_data, "close_remote_data");
156+
m.def("get_buffer_size", &get_buffer_size, "get_buffer_size", py::call_guard<py::gil_scoped_release>());
157+
m.def("get_data_ptr", &get_data_ptr, "get_data_ptr", py::call_guard<py::gil_scoped_release>());
158+
m.def("get_remote_data_ptr", &get_remote_data_ptr, "get_remote_data_ptr", py::call_guard<py::gil_scoped_release>());
159+
m.def("close_remote_data", &close_remote_data, "close_remote_data", py::call_guard<py::gil_scoped_release>());
160160

161-
m.def("bn_fwd_nhwc", &nhwc_bn_fwd_train, "bn_fwd_nhwc");
162-
m.def("bn_fwd_eval_nhwc", &nhwc_bn_fwd_eval, "bn_fwd_eval_nhwc");
163-
m.def("bn_bwd_nhwc", &nhwc_bn_bwd, "bn_bwd_nhwc");
161+
m.def("bn_fwd_nhwc", &nhwc_bn_fwd_train, "bn_fwd_nhwc", py::call_guard<py::gil_scoped_release>());
162+
m.def("bn_fwd_eval_nhwc", &nhwc_bn_fwd_eval, "bn_fwd_eval_nhwc", py::call_guard<py::gil_scoped_release>());
163+
m.def("bn_bwd_nhwc", &nhwc_bn_bwd, "bn_bwd_nhwc", py::call_guard<py::gil_scoped_release>());
164164

165-
m.def("bn_fwd_nhwc_occupancy", &nhwc_bn_fwd_occupancy, "bn_fwd_nhwc_occupancy");
166-
m.def("bn_bwd_nhwc_occupancy", &nhwc_bn_bwd_occupancy, "bn_bwd_nhwc_occupancy");
165+
m.def("bn_fwd_nhwc_occupancy", &nhwc_bn_fwd_occupancy, "bn_fwd_nhwc_occupancy", py::call_guard<py::gil_scoped_release>());
166+
m.def("bn_bwd_nhwc_occupancy", &nhwc_bn_bwd_occupancy, "bn_bwd_nhwc_occupancy", py::call_guard<py::gil_scoped_release>());
167167

168-
m.def("bn_addrelu_fwd_nhwc", &nhwc_bn_addrelu_fwd_train, "bn_addrelu_fwd_nhwc");
169-
m.def("bn_addrelu_fwd_eval_nhwc", &nhwc_bn_addrelu_fwd_eval, "bn_addrelu_fwd_eval_nhwc");
170-
m.def("bn_addrelu_bwd_nhwc", &nhwc_bn_addrelu_bwd, "bn_addrelu_bwd_nhwc");
168+
m.def("bn_addrelu_fwd_nhwc", &nhwc_bn_addrelu_fwd_train, "bn_addrelu_fwd_nhwc", py::call_guard<py::gil_scoped_release>());
169+
m.def("bn_addrelu_fwd_eval_nhwc", &nhwc_bn_addrelu_fwd_eval, "bn_addrelu_fwd_eval_nhwc", py::call_guard<py::gil_scoped_release>());
170+
m.def("bn_addrelu_bwd_nhwc", &nhwc_bn_addrelu_bwd, "bn_addrelu_bwd_nhwc", py::call_guard<py::gil_scoped_release>());
171171

172-
m.def("bn_addrelu_fwd_nhwc_occupancy", &nhwc_bn_addrelu_fwd_occupancy, "bn_addrelu_fwd_nhwc_occupancy");
173-
m.def("bn_addrelu_bwd_nhwc_occupancy", &nhwc_bn_addrelu_bwd_occupancy, "bn_addrelu_bwd_nhwc_occupancy");
172+
m.def("bn_addrelu_fwd_nhwc_occupancy", &nhwc_bn_addrelu_fwd_occupancy, "bn_addrelu_fwd_nhwc_occupancy", py::call_guard<py::gil_scoped_release>());
173+
m.def("bn_addrelu_bwd_nhwc_occupancy", &nhwc_bn_addrelu_bwd_occupancy, "bn_addrelu_bwd_nhwc_occupancy", py::call_guard<py::gil_scoped_release>());
174174
}
175175

‎apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -124,16 +124,22 @@ void index_mul_2d_half_backwrad_backward(
124124

125125
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
126126
m.def("float_forward", &index_mul_2d_float_forward,
127-
"index mul float calculation forward (CUDA)");
127+
"index mul float calculation forward (CUDA)",
128+
py::call_guard<py::gil_scoped_release>());
128129
m.def("float_backward", &index_mul_2d_float_backward,
129-
"index mul float calculation backward (CUDA)");
130+
"index mul float calculation backward (CUDA)",
131+
py::call_guard<py::gil_scoped_release>());
130132
m.def("float_backward_backward", &index_mul_2d_float_backwrad_backward,
131-
"index mul float calculation backward backward (CUDA)");
133+
"index mul float calculation backward backward (CUDA)",
134+
py::call_guard<py::gil_scoped_release>());
132135
m.def("half_forward", &index_mul_2d_half_forward,
133-
"index mul half calculation forward (CUDA)");
136+
"index mul half calculation forward (CUDA)",
137+
py::call_guard<py::gil_scoped_release>());
134138
m.def("half_backward", &index_mul_2d_half_backward,
135-
"index mul half calculation backward (CUDA)");
139+
"index mul half calculation backward (CUDA)",
140+
py::call_guard<py::gil_scoped_release>());
136141
m.def("half_backward_backward", &index_mul_2d_half_backwrad_backward,
137-
"index mul half calculation backward backward (CUDA)");
142+
"index mul half calculation backward backward (CUDA)",
143+
py::call_guard<py::gil_scoped_release>());
138144
}
139145

‎apex/contrib/csrc/layer_norm/ln_api.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,6 @@ std::vector<at::Tensor> ln_bwd(const at::Tensor &dz, // BxSxh
252252

253253
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
254254
m.doc() = "CUDA LayerNorm";
255-
m.def("ln_fwd", &ln_fwd, "Run LayerNorm forward kernel");
256-
m.def("ln_bwd", &ln_bwd, "Run LayerNorm backward kernel");
255+
m.def("ln_fwd", &ln_fwd, "Run LayerNorm forward kernel", py::call_guard<py::gil_scoped_release>());
256+
m.def("ln_bwd", &ln_bwd, "Run LayerNorm backward kernel", py::call_guard<py::gil_scoped_release>());
257257
}

‎apex/contrib/csrc/multihead_attn/multihead_attn_frontend.cpp‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -794,40 +794,46 @@ bwd(int heads, torch::Tensor const &output_grads,
794794
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
795795
m.def("additive_mask_softmax_dropout_forward",
796796
&multihead_attn::fused_softmax::additive_mask_softmax_dropout::fwd,
797-
"Self Multihead Attention masked softmax dropout -- Forward.");
797+
"Self Multihead Attention masked softmax dropout -- Forward.",
798+
py::call_guard<py::gil_scoped_release>());
798799
m.def("additive_mask_softmax_dropout_backward",
799800
&multihead_attn::fused_softmax::additive_mask_softmax_dropout::bwd,
800-
"Self Multihead Attention masked softmax dropout -- Backward.");
801+
"Self Multihead Attention masked softmax dropout -- Backward.",
802+
py::call_guard<py::gil_scoped_release>());
801803
m.def("mask_softmax_dropout_forward", &multihead_attn::fused_softmax::mask_softmax_dropout::fwd,
802-
"Self Multihead Attention masked softmax dropout -- Forward.");
804+
"Self Multihead Attention masked softmax dropout -- Forward.",
805+
py::call_guard<py::gil_scoped_release>());
803806
m.def("mask_softmax_dropout_backward", &multihead_attn::fused_softmax::mask_softmax_dropout::bwd,
804-
"Self Multihead Attention masked softmax dropout -- Backward.");
807+
"Self Multihead Attention masked softmax dropout -- Backward.",
808+
py::call_guard<py::gil_scoped_release>());
805809
m.def("encdec_multihead_attn_forward", &multihead_attn::encdec::cublas_gemmex::fwd,
806-
"Encdec Multihead Attention Forward.");
810+
"Encdec Multihead Attention Forward.", py::call_guard<py::gil_scoped_release>());
807811
m.def("encdec_multihead_attn_backward", &multihead_attn::encdec::cublas_gemmex::bwd,
808-
"Encdec Multihead Attention Backward.");
812+
"Encdec Multihead Attention Backward.", py::call_guard<py::gil_scoped_release>());
809813
m.def("encdec_multihead_attn_norm_add_forward", &multihead_attn::encdec_norm_add::cublas_gemmex::fwd,
810-
"Encdec Multihead Attention Plus Layer Norm and Residual Add Forward.");
814+
"Encdec Multihead Attention Plus Layer Norm and Residual Add Forward.",
815+
py::call_guard<py::gil_scoped_release>());
811816
m.def(
812817
"encdec_multihead_attn_norm_add_backward", &multihead_attn::encdec_norm_add::cublas_gemmex::bwd,
813-
"Encdec Multihead Attention Plus Layer Norm and Residual Add Backward.");
818+
"Encdec Multihead Attention Plus Layer Norm and Residual Add Backward.",
819+
py::call_guard<py::gil_scoped_release>());
814820
m.def("self_attn_forward", &multihead_attn::self::cublas_gemmex::fwd,
815-
"Self Multihead Attention Forward.");
821+
"Self Multihead Attention Forward.", py::call_guard<py::gil_scoped_release>());
816822
m.def("self_attn_backward", &multihead_attn::self::cublas_gemmex::bwd,
817-
"Self Multihead Attention Backward.");
823+
"Self Multihead Attention Backward.", py::call_guard<py::gil_scoped_release>());
818824
m.def("self_attn_bias_forward", &multihead_attn::self_bias::cublas_gemmex::fwd,
819-
"Self Multihead Attention with Bias -- Forward.");
825+
"Self Multihead Attention with Bias -- Forward.", py::call_guard<py::gil_scoped_release>());
820826
m.def("self_attn_bias_backward", &multihead_attn::self_bias::cublas_gemmex::bwd,
821-
"Self Multihead Attention with Bias -- Backward.");
827+
"Self Multihead Attention with Bias -- Backward.", py::call_guard<py::gil_scoped_release>());
822828
m.def("self_attn_bias_additive_mask_forward", &multihead_attn::self_bias_additive_mask::cublas_gemmex::fwd,
823-
"Self Multihead Attention with Bias -- Forward.");
829+
"Self Multihead Attention with Bias -- Forward.", py::call_guard<py::gil_scoped_release>());
824830
m.def("self_attn_bias_additive_mask_backward",
825831
&multihead_attn::self_bias_additive_mask::cublas_gemmex::bwd,
826-
"Self Multihead Attention with Bias -- Backward.");
832+
"Self Multihead Attention with Bias -- Backward.", py::call_guard<py::gil_scoped_release>());
827833
m.def("self_attn_norm_add_forward", &multihead_attn::self_norm_add::cublas_gemmex::fwd,
828-
"Self Multihead Attention Plus Layer Norm and Residual Add Forward.");
834+
"Self Multihead Attention Plus Layer Norm and Residual Add Forward.", py::call_guard<py::gil_scoped_release>());
829835
m.def("self_attn_norm_add_backward", &multihead_attn::self_norm_add::cublas_gemmex::bwd,
830-
"Self Multihead Attention Plus Layer Norm and Residual Add Backward.");
836+
"Self Multihead Attention Plus Layer Norm and Residual Add Backward.", py::call_guard<py::gil_scoped_release>());
831837
}
832838

833839
#undef CHECK_CUDA

0 commit comments

Comments
 (0)