#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) #pragma once #ifdef USE_ROCM // Expect to be included after headers of at::zeros_like and at::empty_like #include #include #include #include #include #include #if AOTRITON_VERSION_CURRENT >= AOTRITON_VERSION_INT(0, 12) #define AOTRITON_V2_API_FLASH_ATTN_H // Suppress the include of deprecated flash/v2.h #endif //////////////////////////////////////////////////////////////////////////////// // Common macros copied from cuda/mem_eff_attention/gemm_kernel_utils.h //////////////////////////////////////////////////////////////////////////////// namespace sdp { namespace aotriton_adapter { inline aotriton::DType cast_dtype(caffe2::TypeMeta t_dtype) { #define CAST_TYPE(aname, dtname) if (t_dtype == at::aname) return aotriton::DType::dtname CAST_TYPE(kByte, kUInt8); CAST_TYPE(kUInt16, kUInt16); CAST_TYPE(kUInt32, kUInt32); CAST_TYPE(kUInt64, kUInt64); CAST_TYPE(kChar, kInt8); CAST_TYPE(kShort, kInt16); CAST_TYPE(kInt, kInt32); CAST_TYPE(kLong, kInt64); CAST_TYPE(kHalf, kFloat16); CAST_TYPE(kFloat, kFloat32); CAST_TYPE(kBFloat16, kBFloat16); return aotriton::DType::kUnknown; #undef CAST_TYPE } template struct IntArrayRefCaster { // std::array cast(IntArrayRef); }; template struct IntArrayRefCaster { static auto cast(at::IntArrayRef ref) { return std::array{{ static_cast(ref.at(0)) }}; } }; template struct IntArrayRefCaster { static auto cast(at::IntArrayRef ref) { return std::array{{ static_cast(ref.at(0)), static_cast(ref.at(1)) }}; } }; template struct IntArrayRefCaster { static auto cast(at::IntArrayRef ref) { return std::array{{ static_cast(ref.at(0)), static_cast(ref.at(1)), static_cast(ref.at(2)) }}; } }; template struct IntArrayRefCaster { static auto cast(at::IntArrayRef ref) { return std::array{{ static_cast(ref.at(0)), static_cast(ref.at(1)), static_cast(ref.at(2)), static_cast(ref.at(3)) }}; } }; template aotriton::TensorView mk_aotensor(const at::Tensor& q, std::string_view tensor_name) { const auto strides = q.strides(); int real_rank = strides.size(); if (real_rank != Rank) { // Lazy conversion of tensor_name TORCH_CHECK(false, std::string(tensor_name) + "'s rank should be " + std::to_string(Rank) + " but is " + std::to_string(real_rank)); } return aotriton::TensorView(reinterpret_cast(q.data_ptr()), IntArrayRefCaster::cast(q.sizes()), IntArrayRefCaster::cast(strides), cast_dtype(q.dtype())); } inline aotriton::TensorView<0> mk_aoscalartensor(const at::Tensor& q) { return aotriton::TensorView<0>(reinterpret_cast(q.data_ptr()), cast_dtype(q.dtype())); } inline aotriton::TensorView<0> mk_philoxtensor(const int64_t* ptr) { return aotriton::TensorView<0>(reinterpret_cast(ptr), aotriton::DType::kUInt64); // AOTriton accepts unsigned int64 } inline aotriton::TensorView<0> mk_atomictensor(const int32_t* ptr) { return aotriton::TensorView<0>(reinterpret_cast(ptr), aotriton::DType::kInt32); } #if AOTRITON_VERSION_CURRENT >= AOTRITON_VERSION_INT(0, 11) struct LazyTensorContext { at::Tensor like_tensor; std::string_view tensor_name; at::Tensor tensor; }; template struct LazyTensorFunctions : public LazyTensorContext { #if AOTRITON_VERSION_CURRENT >= AOTRITON_VERSION_INT(0, 12) using HolderType = aotriton::LazyTensor; #else using HolderType = void; #endif static aotriton::TensorView acquire(HolderType* self) { #if AOTRITON_VERSION_CURRENT >= AOTRITON_VERSION_INT(0, 12) auto ctx = (LazyTensorContext*)self->cookie; #else auto ctx = (LazyTensorContext*)cookie; #endif if (!ctx->tensor.defined()) { auto q = ctx->like_tensor; if constexpr (kRequireZeros) { ctx->tensor = at::zeros(q.sizes(), q.options().dtype(at::kFloat)); } else { ctx->tensor = at::empty_like(q); } } return mk_aotensor(ctx->tensor, ctx->tensor_name); } static void dispose(HolderType* cookie) { } }; template aotriton::LazyTensor mklazy_common(LazyTensorContext* cookie) { using LTF = LazyTensorFunctions; return aotriton::LazyTensor { .cookie = cookie, .acquire = <F::acquire, .dispose = <F::dispose }; } template auto mklazy_empty_like(LazyTensorContext* cookie) { return mklazy_common(cookie); } // Note: this will not keep the original strides template auto mklazy_fp32zeros(LazyTensorContext* cookie) { return mklazy_common(cookie); } inline auto parse_window_size(std::optional window_size_left, std::optional window_size_right) { const int fa_left = window_size_left.value_or(-1); const int fa_right = window_size_right.value_or(-1); auto get_window_value = [](const int window) -> std::optional { if (window < 0) { return std::nullopt; } return window; }; const auto window_left = get_window_value(fa_left); const auto window_right = get_window_value(fa_right); return std::make_tuple(window_left, window_right); } #endif // >= 0.11 } // namespace aotriton_adapter } // namespace sdp namespace at::native { inline int64_t ceil_div(int64_t numerator, int64_t denominator) { return (numerator + (denominator - 1)) / denominator; } } #endif // USE_ROCM #else #error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." #endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION)