|
| 1 | +/****************************************************************************** |
| 2 | + * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. |
| 3 | + * |
| 4 | + * Redistribution and use in source and binary forms, with or without |
| 5 | + * modification, are permitted provided that the following conditions are met: |
| 6 | + * * Redistributions of source code must retain the above copyright |
| 7 | + * notice, this list of conditions and the following disclaimer. |
| 8 | + * * Redistributions in binary form must reproduce the above copyright |
| 9 | + * notice, this list of conditions and the following disclaimer in the |
| 10 | + * documentation and/or other materials provided with the distribution. |
| 11 | + * * Neither the name of the NVIDIA CORPORATION nor the |
| 12 | + * names of its contributors may be used to endorse or promote products |
| 13 | + * derived from this software without specific prior written permission. |
| 14 | + * |
| 15 | + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 16 | + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 17 | + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 18 | + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY |
| 19 | + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| 20 | + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 21 | + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 22 | + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 23 | + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 24 | + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 25 | + * |
| 26 | + ******************************************************************************/ |
| 27 | + |
| 28 | +#include <torch/extension.h> |
| 29 | +#include <ATen/cuda/CUDAContext.h> |
| 30 | +#include <ATen/Dispatch.h> |
| 31 | + |
| 32 | +constexpr int block_size = 512; |
| 33 | +constexpr int ctas_per_sm = 4; |
| 34 | + |
| 35 | +template <typename scalar_t> |
| 36 | +__global__ void |
| 37 | +__launch_bounds__(block_size) |
| 38 | +mha_fill_kernel(scalar_t* out_tensor, |
| 39 | + const int32_t* const start_row, |
| 40 | + const size_t num_rows) { |
| 41 | + size_t row_stride = gridDim.y * blockDim.x; |
| 42 | + size_t row_index = blockIdx.x + (size_t)start_row[0]; |
| 43 | + size_t col_index = blockIdx.y * blockDim.x + threadIdx.x; |
| 44 | + while (row_index < num_rows) { |
| 45 | + out_tensor[row_index*row_stride + col_index] = 0; |
| 46 | + row_index += gridDim.x; |
| 47 | + } |
| 48 | +} |
| 49 | + |
| 50 | +at::Tensor & mha_fill(at::Tensor &self, const at::Tensor &start_index) { |
| 51 | + auto max_tokens = self.size(0); |
| 52 | + auto self_2d = self.view({max_tokens, -1}); |
| 53 | + auto fcd_size = self_2d.size(1); |
| 54 | + TORCH_CHECK (self.is_contiguous(), "input not contiguous"); |
| 55 | + TORCH_CHECK (fcd_size % block_size == 0, "input size not aligned to block size"); |
| 56 | + const int num_mp = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; |
| 57 | + uint64_t num_blk_y = (uint64_t)(fcd_size / block_size); |
| 58 | + uint64_t num_blk_x = (uint64_t)std::ceil(num_mp * ctas_per_sm / num_blk_y); |
| 59 | + dim3 dim_grid(num_blk_x, num_blk_y); |
| 60 | + dim3 dim_block(block_size); |
| 61 | + |
| 62 | + AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2( |
| 63 | + at::ScalarType::Half, at::ScalarType::BFloat16, self_2d.scalar_type(), "mha_padding_fill_", [&]() { |
| 64 | + mha_fill_kernel<<<dim_grid, dim_block, 0, at::cuda::getCurrentCUDAStream()>>>( |
| 65 | + self_2d.data_ptr<scalar_t>(), start_index.data_ptr<int32_t>(), max_tokens); |
| 66 | + C10_CUDA_KERNEL_LAUNCH_CHECK(); |
| 67 | + }); |
| 68 | + return self; |
| 69 | +} |
0 commit comments